framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,64,128,1,float16,float16,0,8.70803705851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,64,128,1,float16,fp8,0,8.674959818522135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,64,128,1,fp8,fp8,0,10.892144521077475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,64,0,1,float16,float16,0,56.890787760416664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,64,128,1,float16,float16,0,9.033648173014322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,64,0,1,fp8,fp8,0,51.21941121419271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,64,0,1,float16,fp8,0,56.08086649576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,64,128,1,float16,fp8,0,8.976629257202148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,64,128,1,float16,float16,0,9.10978635152181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,64,128,1,fp8,fp8,0,11.253835042317709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,64,128,1,float16,fp8,0,9.033418655395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,64,128,1,fp8,fp8,0,11.255855560302734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,64,128,1,float16,float16,0,9.075770696004232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,64,128,1,float16,fp8,0,9.019509633382162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,64,0,1,fp8,fp8,0,51.78791809082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,64,128,1,fp8,fp8,0,11.285530090332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,64,0,1,float16,float16,0,56.725118001302086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,64,0,1,float16,fp8,0,56.88995869954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,64,0,1,float16,float16,0,58.774810791015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,64,0,1,fp8,fp8,0,51.73983256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,64,128,1,float16,float16,0,5.171770731608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,64,128,1,float16,fp8,0,5.078997294108073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,64,0,1,float16,fp8,0,56.43651326497396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,64,128,1,fp8,fp8,0,6.406026840209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,64,128,1,float16,float16,0,4.318832079569499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,64,0,1,float16,float16,0,57.61329142252604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,64,128,1,float16,fp8,0,4.261749267578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,64,128,1,fp8,fp8,0,5.556912104288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,64,0,1,float16,float16,0,29.385963439941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,64,0,1,fp8,fp8,0,26.612592061360676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,64,128,1,float16,float16,0,4.589088122049968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,64,0,1,float16,fp8,0,29.141563415527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,64,0,1,float16,float16,0,28.074058532714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,64,0,1,fp8,fp8,0,51.59368387858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,64,128,1,float16,fp8,0,4.452186584472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,64,128,1,fp8,fp8,0,5.749392191569011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,64,0,1,float16,fp8,0,58.07484436035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,64,0,1,fp8,fp8,0,25.515118916829426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,64,128,1,float16,float16,0,4.499615987141927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,64,0,1,float16,fp8,0,29.176607767740887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,64,128,1,float16,fp8,0,4.462106704711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,64,0,1,float16,float16,0,28.715855916341145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,64,128,1,fp8,fp8,0,5.7484480539957685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,64,0,1,float16,fp8,0,28.182459513346355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,64,128,1,float16,float16,0,4.540581385294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,64,0,1,fp8,fp8,0,25.824452718098957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,64,128,1,float16,fp8,0,4.501215934753418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,64,0,1,float16,float16,0,28.428080240885418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,64,128,1,fp8,fp8,0,5.75540288289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,64,0,1,fp8,fp8,0,25.708719889322918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,64,0,1,float16,fp8,0,28.474021911621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,64,128,1,float16,fp8,0,2.526858647664388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,64,128,1,float16,float16,0,2.5645012855529785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,64,128,1,fp8,fp8,0,3.207162539164225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,64,0,1,float16,float16,0,28.955647786458332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,64,128,1,float16,float16,0,2.305546601613363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,64,0,1,float16,float16,0,14.443456013997396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,64,128,1,float16,fp8,0,2.350554625193278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,64,128,1,fp8,fp8,0,2.907029469807943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,64,0,1,fp8,fp8,0,25.891754150390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,64,0,1,fp8,fp8,0,13.284788767496744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,64,0,1,float16,fp8,0,14.584885915120443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,64,0,1,float16,fp8,0,28.6212158203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,64,0,1,float16,float16,0,14.225755055745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,64,128,1,float16,float16,0,2.296661376953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,64,128,1,float16,fp8,0,2.2657440503438315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,64,0,1,float16,fp8,0,15.392901102701822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,64,128,1,fp8,fp8,0,2.9111948013305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,64,0,1,fp8,fp8,0,12.962020874023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,64,128,1,float16,float16,0,2.3289972941080728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,64,0,1,float16,float16,0,14.596548716227213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,64,128,1,float16,fp8,0,2.2805652618408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,64,128,1,fp8,fp8,0,2.9300266901652017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,64,128,1,float16,float16,0,2.302821318308512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,64,0,1,float16,fp8,0,14.27022933959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,64,0,1,fp8,fp8,0,12.793328603108725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,64,0,1,float16,float16,0,14.238250732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,64,128,1,float16,fp8,0,2.2850027084350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,64,0,1,fp8,fp8,0,12.959786732991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,64,0,1,float16,fp8,0,14.19772211710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,64,0,1,float16,float16,0,14.290629069010416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,64,128,1,float16,float16,0,1.6198293368021648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,64,128,1,float16,fp8,0,1.6154026985168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,64,128,1,fp8,fp8,0,2.9482507705688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,64,128,1,fp8,fp8,0,1.9232746760050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,64,128,1,float16,float16,0,1.6089706420898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,64,0,1,float16,fp8,0,14.330960591634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,64,0,1,float16,float16,0,7.612842559814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,64,128,1,float16,fp8,0,1.6084267298380535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,64,128,1,fp8,fp8,0,1.915818691253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,64,0,1,fp8,fp8,0,12.922602335611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,64,0,1,fp8,fp8,0,6.9462025960286455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,64,0,1,float16,fp8,0,7.613909403483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,64,0,1,float16,float16,0,7.615872065226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,64,0,1,float16,fp8,0,7.61956787109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,64,128,1,float16,float16,0,1.607909361521403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,64,128,1,float16,fp8,0,1.6099573771158855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,64,0,1,fp8,fp8,0,6.944426854451497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,64,128,1,fp8,fp8,0,1.9194614092508953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,64,0,1,float16,float16,0,7.613264083862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,64,128,1,float16,float16,0,1.6095360120137532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,64,128,1,float16,fp8,0,1.6110080083211262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,64,128,1,fp8,fp8,0,1.9147626558939617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,64,0,1,float16,fp8,0,7.608949025472005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,64,128,1,float16,float16,0,1.6085707346598308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,64,0,1,fp8,fp8,0,6.942629496256511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,64,128,1,float16,fp8,0,1.6005013783772786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,64,128,1,fp8,fp8,0,1.9149227142333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,64,0,1,float16,float16,0,7.6186879475911455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,64,0,1,fp8,fp8,0,6.952480316162109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,64,0,1,float16,fp8,0,7.610986709594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,64,0,1,float16,float16,0,7.613424301147461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,64,0,1,float16,fp8,0,7.611562728881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,64,0,1,fp8,fp8,0,6.906976064046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,64,128,1,float16,float16,0,6.509685516357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,64,128,1,float16,fp8,0,6.468559900919597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,64,128,1,fp8,fp8,0,8.22388776143392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,64,128,1,float16,float16,0,6.8154557545979815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,64,128,1,float16,fp8,0,6.775327682495117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,64,128,1,fp8,fp8,0,8.429871877034506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,64,128,1,float16,float16,0,6.769231796264648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,64,0,1,fp8,fp8,0,29.78759511311849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,64,0,1,float16,float16,0,36.330912272135414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,64,128,1,float16,fp8,0,6.714672088623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,64,0,1,float16,fp8,0,34.144996643066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,64,128,1,fp8,fp8,0,8.43830426534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,64,0,1,float16,float16,0,37.05618031819662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,64,0,1,fp8,fp8,0,31.542246500651043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,64,128,1,float16,float16,0,6.864677429199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,64,0,1,float16,fp8,0,36.371541341145836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,64,128,1,float16,fp8,0,6.815765380859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,64,128,1,fp8,fp8,0,8.498704274495443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,64,0,1,float16,float16,0,34.97461446126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,64,128,1,float16,float16,0,3.8353598912556968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,64,128,1,float16,fp8,0,3.7330878575642905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,64,128,1,fp8,fp8,0,4.769536018371582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,64,0,1,fp8,fp8,0,31.96105702718099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,64,0,1,float16,fp8,0,37.35380299886068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,64,128,1,float16,float16,0,3.2096052169799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,64,0,1,float16,float16,0,17.120501200358074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,64,128,1,float16,fp8,0,3.1572319666544595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,64,0,1,float16,float16,0,36.66176096598307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,64,0,1,float16,fp8,0,16.99936040242513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,64,0,1,fp8,fp8,0,30.40454864501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,64,128,1,fp8,fp8,0,4.0629119873046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,64,0,1,float16,fp8,0,35.412645975748696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,64,0,1,fp8,fp8,0,15.59674072265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,64,128,1,float16,float16,0,3.2201760609944663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,64,0,1,float16,float16,0,16.618799845377605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,64,128,1,float16,fp8,0,3.2036587397257485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,64,128,1,fp8,fp8,0,4.185274759928386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,64,128,1,float16,float16,0,3.256469408671061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,64,0,1,fp8,fp8,0,14.920618693033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,64,128,1,float16,fp8,0,3.208127975463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,64,0,1,float16,fp8,0,16.29295476277669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,64,128,1,fp8,fp8,0,4.186293284098308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,64,128,1,float16,float16,0,3.2435146967569985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,64,0,1,float16,float16,0,16.75938669840495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,64,0,1,float16,fp8,0,17.29225031534831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,64,0,1,fp8,fp8,0,14.97092310587565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,64,128,1,float16,fp8,0,3.2270720799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,64,128,1,fp8,fp8,0,4.232607841491699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,64,0,1,float16,float16,0,16.472357432047527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,64,128,1,float16,float16,0,1.885024070739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,64,0,1,fp8,fp8,0,14.930623372395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,64,128,1,float16,fp8,0,1.8947146733601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,64,0,1,float16,fp8,0,16.93564224243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,64,128,1,fp8,fp8,0,2.3956106503804526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,64,128,1,float16,float16,0,1.7236746152242024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,64,0,1,float16,float16,0,16.59050114949544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,64,0,1,float16,float16,0,8.600298563639322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,64,128,1,float16,fp8,0,1.695034662882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,64,0,1,fp8,fp8,0,7.82316780090332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,64,128,1,fp8,fp8,0,2.18230930964152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,64,0,1,float16,fp8,0,17.378799438476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,64,0,1,float16,float16,0,8.313546498616537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,64,0,1,float16,fp8,0,8.519584019978842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,64,0,1,fp8,fp8,0,15.0349489847819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,64,128,1,float16,float16,0,1.709221363067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,64,128,1,float16,fp8,0,1.69978666305542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,64,128,1,fp8,fp8,0,2.158741315205892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,64,0,1,float16,fp8,0,8.475503921508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,64,0,1,fp8,fp8,0,7.607536315917969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,64,128,1,float16,float16,0,1.7246880531311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,64,128,1,float16,fp8,0,1.694485346476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,64,128,1,fp8,fp8,0,2.189477284749349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,64,0,1,float16,float16,0,8.47556241353353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,64,0,1,fp8,fp8,0,7.6114349365234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,64,0,1,float16,fp8,0,8.305775960286459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,64,128,1,float16,float16,0,1.7373919486999512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,64,0,1,float16,float16,0,8.35373306274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,64,128,1,float16,fp8,0,1.7159892717997234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,64,128,1,fp8,fp8,0,2.1679627100626626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,64,128,1,float16,float16,0,1.2098453044891357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,64,0,1,fp8,fp8,0,7.624229431152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,64,0,1,float16,fp8,0,8.324362436930338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,64,128,1,float16,fp8,0,1.2107679843902588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,64,0,1,float16,float16,0,8.430896123250326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,64,0,1,float16,float16,0,4.746506690979004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,64,128,1,fp8,fp8,0,1.445631980895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,64,0,1,fp8,fp8,0,7.573141098022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,64,128,1,float16,fp8,0,1.2172533671061199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,64,0,1,float16,fp8,0,8.314191818237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,64,128,1,float16,float16,0,1.213263988494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,64,0,1,float16,fp8,0,4.684176127115886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,64,0,1,fp8,fp8,0,4.193610509236653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,64,0,1,float16,float16,0,4.627626736958821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,64,128,1,fp8,fp8,0,1.4457440376281738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,64,0,1,float16,fp8,0,4.541962623596191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,64,128,1,float16,float16,0,1.2209333578745525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,64,0,1,fp8,fp8,0,4.22493330637614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,64,128,1,float16,fp8,0,1.2171626885732014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,64,0,1,float16,float16,0,4.607845306396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,64,128,1,fp8,fp8,0,1.445674737294515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,64,0,1,float16,fp8,0,4.540378570556641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,64,0,1,fp8,fp8,0,4.184144020080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,64,128,1,float16,float16,0,1.216864029566447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,64,128,1,float16,fp8,0,1.2165439923604329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,64,128,1,fp8,fp8,0,1.4496960639953613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,64,0,1,float16,float16,0,4.541221300760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,64,128,1,float16,float16,0,1.2175947030385335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,64,0,1,float16,fp8,0,4.537050565083821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,64,0,1,fp8,fp8,0,4.187301317850749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,64,0,1,fp8,fp8,0,4.1940053304036455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,64,128,1,float16,fp8,0,1.2133440176645915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,64,128,1,float16,float16,0,5.3410186767578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,64,128,1,fp8,fp8,0,1.4464640617370605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,64,0,1,float16,float16,0,4.539360046386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,64,0,1,float16,fp8,0,4.591615994771321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,64,128,1,float16,fp8,0,5.279754638671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,64,128,1,float16,float16,0,5.651818593343099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,64,128,1,fp8,fp8,0,6.778863906860352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,64,128,1,float16,fp8,0,5.620789210001628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,64,128,1,fp8,fp8,0,7.06059201558431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,64,128,1,float16,float16,0,5.658090591430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,64,0,1,float16,float16,0,23.432884216308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,64,0,1,float16,fp8,0,23.178993225097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,64,0,1,fp8,fp8,0,21.24413299560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,64,0,1,float16,float16,0,23.5425287882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,64,128,1,float16,fp8,0,5.624874750773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,64,0,1,fp8,fp8,0,21.505343119303387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,64,0,1,float16,fp8,0,23.502756754557293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,64,128,1,fp8,fp8,0,7.113973617553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,64,128,1,float16,float16,0,5.700426737467448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,64,0,1,float16,float16,0,23.929898579915363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,64,128,1,float16,fp8,0,5.6733442942301435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,64,128,1,fp8,fp8,0,7.11294428507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,64,128,1,float16,float16,0,3.1586879094441733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,64,0,1,fp8,fp8,0,21.48138682047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,64,0,1,float16,fp8,0,23.712015787760418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,64,128,1,float16,fp8,0,3.024261474609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,64,128,1,fp8,fp8,0,3.9704106648763022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,64,128,1,float16,float16,0,2.662890593210856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,64,0,1,float16,float16,0,12.173146565755209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,64,0,1,float16,float16,0,23.76012674967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,64,0,1,fp8,fp8,0,21.44951883951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,64,0,1,float16,fp8,0,13.091871897379557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,64,0,1,fp8,fp8,0,11.225285847981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,64,0,1,float16,fp8,0,23.75055440266927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,64,128,1,float16,fp8,0,2.6296960512797036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,64,128,1,float16,float16,0,2.702224095662435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,64,128,1,fp8,fp8,0,3.421504020690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,64,0,1,float16,float16,0,11.730677286783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,64,0,1,float16,fp8,0,11.69058609008789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,64,128,1,float16,fp8,0,2.6586400667826333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,64,0,1,fp8,fp8,0,10.625722885131836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,64,128,1,fp8,fp8,0,3.454335848490397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,64,0,1,float16,float16,0,11.684453328450521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,64,128,1,float16,float16,0,2.7080586751302085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,64,128,1,float16,fp8,0,2.6680854161580405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,64,0,1,float16,fp8,0,11.618459065755209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,64,0,1,fp8,fp8,0,10.653050740559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,64,128,1,fp8,fp8,0,3.4635254542032876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,64,0,1,float16,float16,0,11.747467041015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,64,128,1,float16,float16,0,2.7193705240885415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,64,128,1,float16,fp8,0,2.68721071879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,64,0,1,float16,fp8,0,11.64352544148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,64,0,1,fp8,fp8,0,10.675210316975912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,64,128,1,fp8,fp8,0,3.5194880167643228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,64,0,1,float16,float16,0,11.703018188476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,64,128,1,float16,fp8,0,1.6018613179524739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,64,0,1,float16,fp8,0,11.623461405436197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,64,0,1,float16,float16,0,6.236810684204102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,64,128,1,float16,float16,0,1.5909013748168945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,64,128,1,fp8,fp8,0,2.016981283823649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,64,0,1,fp8,fp8,0,10.66594123840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,64,128,1,float16,float16,0,1.441263993581136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,64,0,1,float16,fp8,0,6.137461344401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,64,0,1,fp8,fp8,0,5.652032216389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,64,128,1,float16,fp8,0,1.4188960393269856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,64,0,1,float16,float16,0,6.344704310099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,64,128,1,fp8,fp8,0,1.835850715637207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,64,128,1,float16,float16,0,1.4618026415507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,64,0,1,float16,fp8,0,5.989930470784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,64,0,1,fp8,fp8,0,5.452053070068359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,64,128,1,float16,float16,0,1.4539626439412434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,64,128,1,float16,fp8,0,1.4301172892252605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,64,0,1,float16,float16,0,6.023290634155273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,64,128,1,fp8,fp8,0,1.836832046508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,64,0,1,float16,fp8,0,5.939066569010417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,64,0,1,fp8,fp8,0,5.46397336324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,64,128,1,float16,fp8,0,1.4336640040079753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,64,128,1,fp8,fp8,0,1.8412480354309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,64,0,1,float16,float16,0,6.059306462605794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,64,128,1,float16,float16,0,1.4603360493977864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,64,0,1,fp8,fp8,0,5.468751907348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,64,0,1,float16,fp8,0,5.9594987233479815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,64,0,1,float16,float16,0,5.981493631998698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,64,128,1,float16,fp8,0,1.4392159779866536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,64,128,1,fp8,fp8,0,1.8526560465494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,64,128,1,float16,float16,0,1.0161333084106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,64,0,1,float16,fp8,0,6.092885335286458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,64,128,1,float16,fp8,0,1.0161279837290447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,64,0,1,fp8,fp8,0,5.429215749104817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,64,0,1,float16,float16,0,3.292538642883301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,64,128,1,float16,fp8,0,1.0156266689300537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,64,0,1,float16,float16,0,3.296016057332357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,64,128,1,fp8,fp8,0,1.2102826436360676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,64,0,1,float16,fp8,0,3.292698542277018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,64,128,1,float16,float16,0,1.0146613121032715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,64,0,1,fp8,fp8,0,3.075050671895345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,64,128,1,fp8,fp8,0,1.2113333543141682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,64,128,1,float16,float16,0,1.0158027013142903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,64,0,1,float16,fp8,0,3.2955945332845054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,64,0,1,fp8,fp8,0,3.046597480773926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,64,128,1,float16,fp8,0,1.018496036529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,64,0,1,float16,float16,0,3.322373390197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,64,128,1,float16,fp8,0,1.0190613269805908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,64,0,1,float16,float16,0,3.2966718673706055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,64,128,1,fp8,fp8,0,1.209818681081136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,64,0,1,float16,fp8,0,3.3236799240112305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,64,0,1,fp8,fp8,0,3.042666753133138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,64,128,1,float16,float16,0,1.018229325612386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,64,128,1,fp8,fp8,0,1.2100693384806316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,64,128,1,float16,float16,0,1.0152053038279216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,64,0,1,float16,fp8,0,3.293951988220215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,64,0,1,fp8,fp8,0,3.0483519236246743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,64,128,1,float16,fp8,0,1.018496036529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,64,128,1,fp8,fp8,0,1.210863987604777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,64,0,1,float16,float16,0,3.297306696573893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,64,0,1,float16,fp8,0,3.3115574518839517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,64,0,1,fp8,fp8,0,3.0483360290527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,64,128,1,float16,float16,0,8.625130971272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,64,128,1,float16,fp8,0,8.562938690185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,64,128,1,fp8,fp8,0,10.71456527709961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,64,128,1,float16,float16,0,8.989775975545248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,64,128,1,float16,fp8,0,8.877834955851236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,64,128,1,fp8,fp8,0,11.15230941772461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,64,128,1,float16,float16,0,8.908554712931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,64,0,1,float16,float16,0,32.57877858479818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,64,0,1,fp8,fp8,0,28.232645670572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,64,0,1,float16,fp8,0,31.632614135742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,64,128,1,float16,fp8,0,8.845562616984049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,64,0,1,float16,float16,0,32.345743815104164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,64,128,1,fp8,fp8,0,11.264165242513021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,64,0,1,fp8,fp8,0,29.324539184570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,64,0,1,float16,fp8,0,33.2372792561849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,64,128,1,float16,float16,0,8.959920247395834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,64,128,1,float16,fp8,0,8.940426508585611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,64,0,1,float16,float16,0,34.28129577636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,64,128,1,fp8,fp8,0,11.265370686848959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,64,128,1,float16,float16,0,5.078117370605469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,64,128,1,float16,fp8,0,4.955077489217122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,64,0,1,fp8,fp8,0,28.62622324625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,64,0,1,float16,fp8,0,31.67162577311198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,64,128,1,fp8,fp8,0,6.301573435465495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,64,0,1,float16,float16,0,16.582784016927082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,64,128,1,float16,float16,0,4.1308746337890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,64,0,1,float16,float16,0,31.81818135579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,64,0,1,fp8,fp8,0,29.27880096435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,64,128,1,float16,fp8,0,4.117786725362142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,64,128,1,fp8,fp8,0,5.380832036336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,64,0,1,float16,fp8,0,32.30982462565104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,64,0,1,float16,fp8,0,16.363455454508465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,64,0,1,fp8,fp8,0,15.241082509358725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,64,128,1,float16,float16,0,4.390384038289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,64,0,1,float16,float16,0,15.527786254882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,64,128,1,float16,fp8,0,4.367717425028483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,64,128,1,fp8,fp8,0,5.603375752766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,64,128,1,float16,float16,0,4.394426663716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,64,0,1,float16,fp8,0,15.893722534179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,64,0,1,fp8,fp8,0,14.102282206217447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,64,128,1,float16,fp8,0,4.358240127563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,64,0,1,float16,float16,0,15.658303578694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,64,128,1,fp8,fp8,0,5.617727915445964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,64,0,1,fp8,fp8,0,14.333727518717447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,64,0,1,float16,fp8,0,15.760762532552084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,64,128,1,float16,float16,0,4.41753609975179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,64,128,1,float16,fp8,0,4.388367970784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,64,0,1,float16,float16,0,15.587525685628256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,64,128,1,fp8,fp8,0,5.66594123840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,64,128,1,float16,float16,0,2.4684160550435386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,64,0,1,float16,fp8,0,16.241402943929035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,64,0,1,fp8,fp8,0,14.318677266438803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,64,128,1,float16,fp8,0,2.398314634958903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,64,128,1,fp8,fp8,0,3.0920588175455728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,64,0,1,float16,float16,0,8.080511728922525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,64,128,1,float16,float16,0,2.092474619547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,64,0,1,float16,float16,0,17.30466079711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,64,0,1,fp8,fp8,0,14.366048177083334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,64,128,1,float16,fp8,0,2.159488042195638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,64,0,1,float16,fp8,0,16.740367889404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,64,0,1,fp8,fp8,0,7.519952138264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,64,0,1,float16,fp8,0,8.558869043986002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,64,128,1,fp8,fp8,0,2.7147305806477866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,64,0,1,float16,float16,0,7.855322519938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,64,0,1,float16,fp8,0,8.036874771118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,64,128,1,float16,float16,0,2.159605344136556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,64,0,1,fp8,fp8,0,7.057402928670247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,64,128,1,float16,fp8,0,2.1347626050313315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,64,128,1,fp8,fp8,0,2.781402587890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,64,0,1,float16,float16,0,7.759471893310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,64,128,1,float16,float16,0,2.160048007965088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,64,0,1,float16,fp8,0,7.799210866292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,64,128,1,float16,fp8,0,2.12610133488973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,64,0,1,fp8,fp8,0,7.080842971801758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,64,0,1,float16,fp8,0,7.729157129923503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,64,128,1,fp8,fp8,0,2.770549456278483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,64,0,1,float16,float16,0,7.779018402099609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,64,128,1,fp8,fp8,0,2.774362564086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,64,128,1,float16,float16,0,2.16976531346639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,64,128,1,float16,float16,0,1.2859893639882405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,64,0,1,fp8,fp8,0,7.1127573649088545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,64,128,1,float16,fp8,0,2.129535992940267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,64,128,1,fp8,fp8,0,1.620255947113037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,64,0,1,float16,float16,0,7.78541882832845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,64,0,1,float16,fp8,0,7.747264226277669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,64,0,1,fp8,fp8,0,7.082634607950847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,64,128,1,float16,fp8,0,1.2789440155029297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,64,0,1,float16,float16,0,4.314517339070638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,64,128,1,float16,float16,0,1.1635039647420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,64,128,1,fp8,fp8,0,1.4762825965881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,64,0,1,float16,fp8,0,4.124090512593587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,64,0,1,fp8,fp8,0,3.801370620727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,64,128,1,float16,fp8,0,1.155178705851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,64,128,1,float16,fp8,0,1.1494239966074626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,64,0,1,float16,float16,0,4.009178797403972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,64,128,1,float16,float16,0,1.166266679763794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,64,128,1,float16,float16,0,1.1728320121765137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,64,0,1,float16,fp8,0,4.076517422993978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,64,0,1,fp8,fp8,0,3.667109489440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,64,128,1,float16,fp8,0,1.1552800337473552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,64,0,1,float16,float16,0,4.0871734619140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,64,128,1,fp8,fp8,0,1.491594632466634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,64,128,1,fp8,fp8,0,1.4799572626749675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,64,0,1,float16,fp8,0,4.069162686665853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,64,0,1,fp8,fp8,0,3.6671520868937173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,64,0,1,float16,float16,0,4.017317454020183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,64,0,1,float16,float16,0,4.002213478088379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,64,0,1,float16,fp8,0,4.1747252146403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,64,128,1,float16,float16,0,1.1797173023223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,64,0,1,fp8,fp8,0,3.6747681299845376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,64,128,1,float16,fp8,0,1.1562613646189372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,64,128,1,fp8,fp8,0,1.4978826840718586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,64,128,1,fp8,fp8,0,0.9801119963328043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,64,128,1,float16,float16,0,0.8204800287882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,64,0,1,float16,fp8,0,3.9807840983072915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,64,0,1,fp8,fp8,0,3.6883999506632485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,64,0,1,float16,float16,0,2.263807932535807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,64,128,1,float16,fp8,0,0.8207306861877441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,64,0,1,float16,fp8,0,2.2470827102661133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,64,0,1,float16,fp8,0,2.2486400604248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,64,128,1,float16,float16,0,0.816981315612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,64,0,1,fp8,fp8,0,2.102656046549479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,64,0,1,float16,float16,0,2.2505760192871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,64,0,1,float16,float16,0,2.2471680641174316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,64,128,1,float16,fp8,0,0.8165973027547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,64,128,1,fp8,fp8,0,0.9747626781463623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,64,0,1,fp8,fp8,0,2.0854293505350747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,64,128,1,float16,float16,0,0.8176693121592203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,64,128,1,float16,fp8,0,0.8175840377807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,64,128,1,fp8,fp8,0,0.9796266555786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,64,0,1,float16,fp8,0,2.2679306666056314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,64,128,1,float16,float16,0,0.8164587020874023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,64,0,1,fp8,fp8,0,2.0871520042419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,64,0,1,float16,float16,0,2.2499146461486816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,64,0,1,fp8,fp8,0,2.0997279485066733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,64,128,1,float16,fp8,0,0.8198026816050211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,64,128,1,fp8,fp8,0,0.9767626921335856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,64,0,1,float16,fp8,0,2.2507947285970054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,64,128,1,float16,float16,0,0.8173226515452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,64,0,1,float16,float16,0,2.250592072804769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,64,128,1,float16,fp8,0,0.818458636601766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,64,128,1,fp8,fp8,0,0.9752426942189535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,64,0,1,float16,fp8,0,2.250271956125895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,64,0,1,fp8,fp8,0,2.0858559608459473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,64,128,1,float16,float16,0,6.443674723307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,64,128,1,float16,fp8,0,6.387744267781575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,64,128,1,fp8,fp8,0,8.095610936482748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,64,128,1,float16,float16,0,6.741343816121419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,64,128,1,float16,fp8,0,6.659621556599935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,64,128,1,fp8,fp8,0,8.395498911539713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,64,0,1,float16,float16,0,18.509258270263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,64,0,1,fp8,fp8,0,16.809221903483074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,64,0,1,float16,fp8,0,18.43117396036784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,64,128,1,float16,float16,0,6.704928080240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,64,0,1,float16,float16,0,18.812410990397137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,64,128,1,float16,fp8,0,6.6667734781901045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,64,0,1,float16,fp8,0,18.744752248128254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,64,128,1,fp8,fp8,0,8.355695724487305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,64,0,1,fp8,fp8,0,17.175275166829426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,64,128,1,float16,float16,0,6.750933329264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,64,128,1,float16,fp8,0,6.69918950398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,64,0,1,float16,float16,0,18.973765055338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,64,128,1,fp8,fp8,0,8.444517135620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,64,0,1,fp8,fp8,0,17.163733164469402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,64,0,1,float16,fp8,0,19.9289067586263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,64,128,1,float16,float16,0,3.772416114807129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,64,0,1,float16,float16,0,19.161392211914062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,64,128,1,float16,fp8,0,3.6880213419596353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,64,128,1,fp8,fp8,0,4.733818689982097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,64,128,1,float16,float16,0,3.0901705423990884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,64,0,1,float16,float16,0,9.97601572672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,64,128,1,float16,fp8,0,3.019498825073242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,64,0,1,float16,fp8,0,19.052106221516926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,64,0,1,fp8,fp8,0,9.318869272867838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,64,0,1,float16,fp8,0,9.807941436767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,64,0,1,fp8,fp8,0,17.258197784423828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,64,128,1,fp8,fp8,0,3.9898560841878257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,64,0,1,float16,float16,0,9.298533121744791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,64,128,1,float16,float16,0,3.1433226267496743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,64,128,1,float16,fp8,0,3.1116053263346353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,64,128,1,fp8,fp8,0,4.064682642618815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,64,0,1,float16,fp8,0,9.197135925292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,64,0,1,fp8,fp8,0,8.29648526509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,64,128,1,float16,float16,0,3.153813362121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,64,128,1,float16,fp8,0,3.0877812703450522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,64,0,1,float16,float16,0,9.347882588704428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,64,128,1,fp8,fp8,0,4.080431938171387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,64,0,1,fp8,fp8,0,8.45306142171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,64,0,1,float16,fp8,0,9.18446413675944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,64,128,1,float16,float16,0,3.146341323852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,64,0,1,float16,float16,0,9.217173258463541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,64,128,1,float16,fp8,0,3.1085332234700522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,64,128,1,float16,float16,0,1.8418240547180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,64,128,1,fp8,fp8,0,4.066351890563965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,64,0,1,fp8,fp8,0,8.41049575805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,64,0,1,float16,fp8,0,9.167311986287435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,64,128,1,float16,fp8,0,1.8044320742289226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,64,0,1,float16,float16,0,9.358357111612955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,64,128,1,fp8,fp8,0,2.282698631286621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,64,0,1,float16,float16,0,4.979925473531087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,64,128,1,float16,float16,0,1.5998560587565105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,64,0,1,fp8,fp8,0,8.49510383605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,64,0,1,float16,fp8,0,9.198634465535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,64,128,1,float16,fp8,0,1.5748106638590496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,64,0,1,float16,fp8,0,5.018154780069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,64,0,1,fp8,fp8,0,4.4782454172770185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,64,128,1,fp8,fp8,0,2.055295944213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,64,0,1,float16,float16,0,4.738677342732747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,64,128,1,float16,float16,0,1.6099413235982258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,64,0,1,float16,fp8,0,4.642527898152669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,64,0,1,fp8,fp8,0,4.223402659098308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,64,128,1,float16,fp8,0,1.5929172833760579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,64,0,1,float16,float16,0,4.914405186971028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,64,128,1,fp8,fp8,0,2.066277345021566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,64,0,1,float16,fp8,0,4.6477813720703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,64,128,1,float16,float16,0,1.625973383585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,64,128,1,float16,fp8,0,1.5944639841715496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,64,128,1,fp8,fp8,0,2.0788000424702964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,64,0,1,fp8,fp8,0,4.262213389078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,64,128,1,float16,float16,0,1.6281545956929524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,64,128,1,float16,fp8,0,1.6014720598856609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,64,0,1,float16,float16,0,4.7027788162231445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,64,0,1,fp8,fp8,0,4.26586119333903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,64,0,1,float16,fp8,0,4.666234652201335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,64,128,1,float16,fp8,0,0.9471626281738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,64,128,1,fp8,fp8,0,2.078218619028727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,64,0,1,fp8,fp8,0,4.271018664042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,64,128,1,float16,float16,0,0.960208018620809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,64,0,1,float16,fp8,0,2.630634625752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,64,128,1,float16,float16,0,0.8645973205566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,64,0,1,float16,float16,0,4.678336143493652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,64,0,1,float16,float16,0,2.4855093955993652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,64,128,1,float16,fp8,0,0.8759679794311523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,64,128,1,fp8,fp8,0,1.2154826323191326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,64,0,1,float16,fp8,0,4.773647944132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,64,0,1,fp8,fp8,0,2.323088010152181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,64,128,1,fp8,fp8,0,1.1065973440806072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,64,0,1,float16,float16,0,2.408554712931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,64,128,1,float16,float16,0,0.8779679934183756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,64,0,1,float16,fp8,0,2.394341309865316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,64,0,1,fp8,fp8,0,2.2432106335957847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,64,0,1,float16,float16,0,2.419109344482422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,64,128,1,float16,fp8,0,0.8772746721903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,64,0,1,float16,float16,0,2.4180320103963218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,64,128,1,fp8,fp8,0,1.1130239963531494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,64,0,1,float16,fp8,0,2.3970185915629068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,64,0,1,fp8,fp8,0,2.2443413734436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,64,128,1,float16,float16,0,0.8828746477762858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,64,128,1,float16,fp8,0,0.870464007059733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,64,128,1,float16,fp8,0,0.8808853626251221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,64,128,1,fp8,fp8,0,1.1195306777954102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,64,0,1,float16,fp8,0,2.4248693784077964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,64,128,1,float16,float16,0,0.8864266872406006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,64,128,1,float16,float16,0,0.6214346488316854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,64,0,1,fp8,fp8,0,2.219088077545166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,64,0,1,float16,float16,0,1.3951412836710613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,64,128,1,fp8,fp8,0,0.742682695388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,64,0,1,float16,float16,0,2.419818719228109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,64,128,1,fp8,fp8,0,1.1249066988627117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,64,0,1,float16,fp8,0,2.406378746032715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,64,0,1,fp8,fp8,0,2.234111944834391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,64,128,1,float16,fp8,0,0.6211839914321899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,64,0,1,float16,fp8,0,1.3970613479614258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,64,0,1,fp8,fp8,0,1.3047626813252766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,64,128,1,float16,float16,0,0.6187893152236938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,64,128,1,float16,fp8,0,0.6196320056915283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,64,0,1,float16,float16,0,1.404352029164632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,64,128,1,fp8,fp8,0,0.7408746878306071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,64,128,1,float16,float16,0,0.6189653476079305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,64,0,1,float16,fp8,0,1.4015679359436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,64,0,1,fp8,fp8,0,1.3049279848734539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,64,128,1,float16,fp8,0,0.6191946665445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,64,0,1,float16,float16,0,1.4009386698404949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,64,0,1,float16,float16,0,1.40066130956014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,64,128,1,fp8,fp8,0,0.7416266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,64,0,1,float16,fp8,0,1.3983947436014812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,64,0,1,fp8,fp8,0,1.3045066992441814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,64,128,1,float16,float16,0,0.6190773248672485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,64,128,1,float16,fp8,0,0.6201866865158081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,64,128,1,fp8,fp8,0,0.7410346666971842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,64,0,1,float16,fp8,0,1.4023733139038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,64,128,1,float16,float16,0,0.6194666624069214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,64,0,1,float16,fp8,0,1.4016745885213215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,64,0,1,fp8,fp8,0,1.3035786946614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,64,128,1,float16,fp8,0,0.6197973489761353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,64,0,1,float16,float16,0,1.3995520273844402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,64,128,1,fp8,fp8,0,0.7407146294911703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,64,0,1,fp8,fp8,0,1.3064693609873455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,64,128,1,float16,float16,0,8.589402516682943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,64,128,1,float16,fp8,0,8.513823827107748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,64,128,1,fp8,fp8,0,10.660885492960611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,64,128,1,float16,float16,0,8.931461334228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,64,128,1,float16,fp8,0,8.807941436767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,64,0,1,float16,float16,0,18.400938669840496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,64,0,1,fp8,fp8,0,16.656879425048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,64,128,1,fp8,fp8,0,11.024458567301432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,64,0,1,float16,fp8,0,18.379440307617188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,64,0,1,float16,float16,0,18.748368581136067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,64,128,1,float16,float16,0,8.875365575154623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,64,128,1,float16,fp8,0,8.838058471679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,64,0,1,fp8,fp8,0,17.00005849202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,64,0,1,float16,fp8,0,18.76905059814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,64,128,1,fp8,fp8,0,11.050000508626303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,64,128,1,float16,float16,0,8.92641576131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,64,0,1,float16,float16,0,19.71349334716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,64,0,1,fp8,fp8,0,18.016565958658855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,64,128,1,float16,fp8,0,8.86685307820638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,64,0,1,float16,fp8,0,19.440202077229817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,64,128,1,fp8,fp8,0,11.15054448445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,64,128,1,float16,float16,0,5.079744021097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,64,128,1,float16,fp8,0,4.921882629394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,64,128,1,fp8,fp8,0,6.269919713338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,64,0,1,float16,float16,0,18.812938690185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,64,0,1,float16,float16,0,10.01253890991211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,64,0,1,float16,fp8,0,18.749764760335285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,64,128,1,float16,float16,0,4.0763092041015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,64,0,1,float16,fp8,0,9.928192138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,64,128,1,float16,fp8,0,4.047199885050456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,64,0,1,fp8,fp8,0,9.198053359985352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,64,0,1,fp8,fp8,0,17.124900817871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,64,128,1,fp8,fp8,0,5.281530698140462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,64,0,1,float16,float16,0,9.143434524536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,64,128,1,float16,fp8,0,4.327040036519368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,64,0,1,float16,fp8,0,8.97501309712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,64,128,1,float16,float16,0,4.359909375508626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,64,0,1,fp8,fp8,0,8.299445470174154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,64,0,1,float16,float16,0,9.255168279012045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,64,0,1,fp8,fp8,0,8.563418706258139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,64,128,1,fp8,fp8,0,5.562725067138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,64,128,1,float16,float16,0,4.3624693552653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,64,0,1,float16,fp8,0,9.238831837972006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,64,128,1,float16,fp8,0,4.347397486368815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,64,0,1,float16,float16,0,9.274415969848633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,64,128,1,fp8,fp8,0,5.5778452555338545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,64,128,1,float16,float16,0,4.410175959269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,64,128,1,float16,fp8,0,4.379232088724772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,64,0,1,fp8,fp8,0,8.577007929484049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,64,0,1,float16,fp8,0,9.247711817423502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,64,128,1,fp8,fp8,0,5.639445622762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,64,0,1,float16,float16,0,9.312063852945963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,64,128,1,float16,float16,0,2.458245277404785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,64,128,1,float16,fp8,0,2.3856587409973145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,64,0,1,float16,fp8,0,9.280160268147787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,64,0,1,float16,float16,0,4.909658749898274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,64,128,1,fp8,fp8,0,3.059978802998861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,64,0,1,fp8,fp8,0,8.560304005940756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,64,128,1,float16,float16,0,2.0713226000467935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,64,0,1,float16,fp8,0,4.84495480855306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,64,0,1,fp8,fp8,0,4.551402727762858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,64,128,1,float16,fp8,0,2.0306026140848794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,64,0,1,float16,float16,0,4.566480000813802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,64,128,1,fp8,fp8,0,2.64683198928833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,64,128,1,float16,float16,0,2.08351469039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,64,0,1,fp8,fp8,0,4.149504025777181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,64,0,1,float16,fp8,0,4.484346707661946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,64,128,1,float16,fp8,0,2.045717398325602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,64,128,1,fp8,fp8,0,2.660858631134033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,64,0,1,float16,float16,0,4.538832028706868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,64,128,1,float16,float16,0,2.0793280601501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,64,128,1,float16,fp8,0,2.0420692761739097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,64,0,1,float16,fp8,0,4.6543839772542315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,64,128,1,fp8,fp8,0,2.6753921508789062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,64,0,1,fp8,fp8,0,4.1640214920043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,64,0,1,float16,float16,0,4.848848025004069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,64,128,1,float16,float16,0,2.0807092984517417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,64,0,1,float16,fp8,0,4.522421201070149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,64,128,1,float16,fp8,0,2.050831953684489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,64,128,1,fp8,fp8,0,2.6409279505411782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,64,0,1,fp8,fp8,0,4.133690516153972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,64,0,1,float16,float16,0,4.71503480275472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,64,0,1,float16,fp8,0,4.556288083394368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,64,128,1,float16,float16,0,1.2287999788920085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,64,0,1,fp8,fp8,0,4.176197369893392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,64,0,1,float16,fp8,0,2.462378660837809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,64,0,1,float16,float16,0,2.4679253896077475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,64,0,1,float16,float16,0,2.3242293993631997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,64,128,1,float16,fp8,0,1.214186668395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,64,128,1,fp8,fp8,0,1.5231413841247559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,64,0,1,fp8,fp8,0,2.285642623901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,64,128,1,float16,float16,0,1.0799466768900554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,64,128,1,float16,fp8,0,1.0634026527404785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,64,128,1,fp8,fp8,0,1.3822827339172363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,64,0,1,float16,fp8,0,2.32152525583903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,64,128,1,float16,float16,0,1.095685323079427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,64,0,1,fp8,fp8,0,2.131722609202067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,64,0,1,float16,float16,0,2.3240639368693032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,64,128,1,float16,fp8,0,1.0699040095011394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,64,128,1,fp8,fp8,0,1.3901599248250325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,64,0,1,float16,fp8,0,2.3084266980489097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,64,128,1,float16,float16,0,1.0892000198364258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,64,0,1,fp8,fp8,0,2.1432852745056152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,64,128,1,fp8,fp8,0,1.3878026008605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,64,128,1,float16,fp8,0,1.0718239943186443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,64,0,1,float16,float16,0,2.3494879404703775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,64,128,1,float16,float16,0,1.095306634902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,64,0,1,fp8,fp8,0,2.1411306063334146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,64,0,1,float16,fp8,0,2.312176068623861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,64,128,1,float16,float16,0,0.6517599821090698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,64,128,1,float16,fp8,0,0.6509173313776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,64,128,1,float16,fp8,0,1.0753280321757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,64,0,1,float16,float16,0,2.3326560656229653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,64,128,1,fp8,fp8,0,1.394330660502116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,64,0,1,float16,float16,0,1.2755893071492512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,64,128,1,fp8,fp8,0,0.8282079696655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,64,0,1,fp8,fp8,0,2.146320025126139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,64,0,1,float16,fp8,0,2.3155412673950195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,64,0,1,float16,fp8,0,1.275861342748006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,64,0,1,fp8,fp8,0,1.2009653250376384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,64,128,1,float16,float16,0,0.5933653513590494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,64,0,1,float16,fp8,0,1.2108746369679768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,64,128,1,float16,fp8,0,0.5864053169886271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,64,0,1,float16,float16,0,1.2237760225931804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,64,0,1,float16,float16,0,1.2225333054860432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,64,128,1,fp8,fp8,0,0.7531253496805826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,64,128,1,float16,float16,0,0.5932159821192423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,64,0,1,fp8,fp8,0,1.1303040186564128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,64,128,1,float16,fp8,0,0.5863626797993978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,64,128,1,fp8,fp8,0,0.752229372660319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,64,0,1,float16,fp8,0,1.2136800289154053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,64,128,1,float16,float16,0,0.5976426601409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,64,0,1,fp8,fp8,0,1.1310293674468994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,64,128,1,float16,fp8,0,0.5887146790822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,64,128,1,fp8,fp8,0,0.7583786646525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,64,0,1,float16,fp8,0,1.2168746789296467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,64,128,1,fp8,fp8,0,0.7608693440755209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,64,0,1,float16,float16,0,1.2262933254241943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,64,128,1,float16,float16,0,0.5963519811630249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,64,128,1,float16,float16,0,0.42443732420603436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,64,0,1,fp8,fp8,0,1.1359253724416096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,64,0,1,fp8,fp8,0,1.1352427005767822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,64,128,1,float16,fp8,0,0.589679996172587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,64,0,1,float16,float16,0,1.228543996810913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,64,0,1,float16,fp8,0,1.2211360136667888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,64,0,1,float16,float16,0,0.7440906365712484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,64,128,1,float16,fp8,0,0.42280534903208417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,64,128,1,fp8,fp8,0,0.508784015973409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,64,0,1,float16,fp8,0,0.7426346937815348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,64,0,1,fp8,fp8,0,0.6801546414693197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,64,128,1,float16,float16,0,0.421066681543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,64,128,1,float16,fp8,0,0.42153600851694745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,64,0,1,float16,float16,0,0.7481119632720947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,64,128,1,fp8,fp8,0,0.505893349647522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,64,0,1,float16,fp8,0,0.7452212969462076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,64,128,1,float16,float16,0,0.4216426610946655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,64,0,1,fp8,fp8,0,0.6763786474863688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,64,128,1,float16,fp8,0,0.4219253460566203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,64,128,1,fp8,fp8,0,0.5070559978485107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,64,0,1,float16,float16,0,0.7434506416320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,64,0,1,fp8,fp8,0,0.6769226392110189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,64,128,1,fp8,fp8,0,0.5057066679000854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,64,128,1,float16,float16,0,0.420906662940979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,64,0,1,float16,fp8,0,0.7442986965179443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,64,128,1,float16,fp8,0,0.4227466583251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,64,0,1,float16,float16,0,0.7437706788380941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,64,0,1,float16,fp8,0,0.7455093065897623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,64,128,1,float16,float16,0,0.4206826686859131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,64,0,1,fp8,fp8,0,0.6764693260192871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,64,0,1,float16,float16,0,0.7458293437957764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,64,128,1,float16,fp8,0,0.4224426746368408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,64,128,1,fp8,fp8,0,0.5051306486129761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,64,0,1,fp8,fp8,0,0.6764533519744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,64,0,1,float16,fp8,0,0.7474026679992676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,64,128,1,float16,float16,0,6.271477381388347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,64,128,1,float16,fp8,0,6.200517018636067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,64,128,1,fp8,fp8,0,8.002485275268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,64,128,1,float16,float16,0,6.455392201741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,64,0,1,float16,float16,0,11.320976257324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,64,0,1,fp8,fp8,0,10.260309219360352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,64,0,1,float16,fp8,0,11.290426890055338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,64,128,1,float16,fp8,0,6.379168192545573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,64,128,1,fp8,fp8,0,7.9737122853597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,64,128,1,float16,float16,0,6.407786687215169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,64,0,1,float16,float16,0,11.493887583414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,64,128,1,float16,fp8,0,6.394416173299153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,64,0,1,float16,fp8,0,11.471989949544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,64,0,1,fp8,fp8,0,10.414917627970377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,64,128,1,fp8,fp8,0,8.125333150227865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,64,0,1,float16,float16,0,11.525850931803385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,64,128,1,float16,float16,0,6.482128143310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,64,0,1,fp8,fp8,0,10.429941177368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,64,0,1,float16,fp8,0,11.487611134847006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,64,128,1,float16,fp8,0,6.4465281168619795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,64,128,1,float16,float16,0,3.719264030456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,64,128,1,fp8,fp8,0,8.223503748575846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,64,0,1,float16,float16,0,11.569759368896484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,64,128,1,float16,fp8,0,3.6489171981811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,64,0,1,float16,fp8,0,11.524826049804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,64,128,1,fp8,fp8,0,4.637141227722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,64,0,1,fp8,fp8,0,10.770757039388021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,64,128,1,float16,float16,0,3.097797393798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,64,0,1,float16,fp8,0,6.3700211842854815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,64,0,1,float16,float16,0,6.277146657307942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,64,0,1,fp8,fp8,0,5.858847935994466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,64,128,1,float16,fp8,0,2.9882081349690757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,64,0,1,float16,float16,0,5.757904052734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,64,128,1,fp8,fp8,0,3.8497279485066733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,64,128,1,float16,float16,0,3.1083733240763345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,64,128,1,float16,fp8,0,3.0407625834147134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,64,0,1,float16,fp8,0,5.531525293986003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,64,0,1,fp8,fp8,0,5.092165311177571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,64,0,1,float16,float16,0,5.633706410725911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,64,128,1,fp8,fp8,0,3.968458811442057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,64,128,1,float16,float16,0,3.1045920054117837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,64,128,1,float16,fp8,0,3.0611359278361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,64,0,1,fp8,fp8,0,5.166080156962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,64,0,1,float16,fp8,0,5.6698455810546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,64,128,1,fp8,fp8,0,3.95304012298584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,64,0,1,float16,float16,0,5.798287709554036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,64,0,1,float16,fp8,0,5.6080322265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,64,128,1,float16,float16,0,3.0984532038370767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,64,0,1,fp8,fp8,0,5.167824109395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,64,128,1,float16,fp8,0,3.062255859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,64,128,1,fp8,fp8,0,4.0373226801554365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,64,0,1,float16,float16,0,5.816383997599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,64,128,1,float16,float16,0,1.8267626762390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,64,128,1,float16,fp8,0,1.7831039428710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,64,0,1,float16,fp8,0,5.62721570332845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,64,0,1,float16,float16,0,3.0878079732259116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,64,128,1,fp8,fp8,0,2.2290026346842446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,64,0,1,fp8,fp8,0,5.232053438822429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,64,128,1,float16,fp8,0,1.517365296681722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,64,0,1,float16,fp8,0,3.0511147181193032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,64,128,1,fp8,fp8,0,1.9844640096028645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,64,0,1,fp8,fp8,0,2.8219305674235025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,64,128,1,float16,float16,0,1.5535626411437988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,64,128,1,float16,fp8,0,1.531450589497884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,64,0,1,float16,float16,0,2.828181266784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,64,128,1,float16,float16,0,1.5412373542785645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,64,0,1,float16,fp8,0,2.8003145853678384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,64,0,1,fp8,fp8,0,2.576709270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,64,0,1,float16,float16,0,2.8224852879842124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,64,128,1,fp8,fp8,0,1.994607925415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,64,128,1,float16,float16,0,1.5630933443705242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,64,0,1,fp8,fp8,0,2.5803893407185874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,64,0,1,float16,fp8,0,2.805370648701986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,64,128,1,float16,fp8,0,1.5404106775919597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,64,0,1,float16,float16,0,2.833589235941569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,64,128,1,fp8,fp8,0,1.9966826438903809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,64,0,1,float16,fp8,0,2.8595253626505532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,64,128,1,float16,float16,0,1.5748480161031086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,64,0,1,fp8,fp8,0,2.586634635925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,64,128,1,fp8,fp8,0,2.01692803700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,64,128,1,float16,fp8,0,1.549957275390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,64,0,1,float16,float16,0,2.868602752685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,64,128,1,float16,float16,0,0.9343199729919434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,64,128,1,float16,fp8,0,0.9160693486531576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,64,128,1,fp8,fp8,0,1.157365322113037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,64,0,1,float16,fp8,0,2.8215786616007485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,64,0,1,float16,float16,0,1.5719680786132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,64,0,1,fp8,fp8,0,2.600714683532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,64,128,1,float16,float16,0,0.8186613718668619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,64,0,1,float16,fp8,0,1.557696024576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,64,0,1,fp8,fp8,0,1.4479626019795735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,64,128,1,float16,fp8,0,0.8008639812469482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,64,128,1,fp8,fp8,0,1.0381279786427815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,64,0,1,float16,float16,0,1.4580480257670085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,64,0,1,float16,fp8,0,1.4454293251037598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,64,0,1,fp8,fp8,0,1.3366773923238118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,64,128,1,float16,float16,0,0.8208853403727213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,64,128,1,float16,fp8,0,0.8064213593800863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,64,128,1,fp8,fp8,0,1.0481173197428386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,64,0,1,float16,float16,0,1.4593173662821453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,64,128,1,float16,float16,0,0.8195040225982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,64,0,1,float16,fp8,0,1.4451893170674641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,64,0,1,fp8,fp8,0,1.3393120765686035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,64,0,1,float16,float16,0,1.4642346700032551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,64,128,1,float16,fp8,0,0.8088693618774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,64,128,1,fp8,fp8,0,1.0505332946777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,64,0,1,float16,fp8,0,1.4509545962015789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,64,128,1,float16,float16,0,0.8234933217366537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,64,0,1,fp8,fp8,0,1.3428373336791992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,64,0,1,float16,float16,0,1.4646347363789876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,64,128,1,float16,fp8,0,0.8142933050791422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,64,128,1,fp8,fp8,0,1.051802635192871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,64,0,1,float16,fp8,0,1.4542826016743977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,64,128,1,float16,float16,0,0.4941279888153076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,64,0,1,fp8,fp8,0,1.3477333386739094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,64,0,1,float16,float16,0,0.8181920051574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,64,0,1,float16,fp8,0,0.8194719950358073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,64,128,1,float16,fp8,0,0.49369064966837567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,64,128,1,fp8,fp8,0,0.6279093424479166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,64,128,1,float16,float16,0,0.4516640106836955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,64,0,1,fp8,fp8,0,0.7765066623687744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,64,128,1,float16,fp8,0,0.44392534097035724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,64,0,1,float16,float16,0,0.7778186798095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,64,128,1,fp8,fp8,0,0.57150932153066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,64,0,1,float16,fp8,0,0.7702186902364095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,64,0,1,fp8,fp8,0,0.7212693691253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,64,128,1,float16,float16,0,0.4526826540629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,64,0,1,float16,float16,0,0.777557373046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,64,128,1,float16,fp8,0,0.44760000705718994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,64,128,1,fp8,fp8,0,0.5743786493937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,64,0,1,float16,fp8,0,0.772320032119751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,64,128,1,float16,float16,0,0.45047998428344727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,64,0,1,fp8,fp8,0,0.7234986623128256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,64,0,1,float16,float16,0,0.7778933048248291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,64,128,1,float16,fp8,0,0.4482239882151286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,64,128,1,fp8,fp8,0,0.5754559834798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,64,128,1,float16,float16,0,0.45449598630269367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,64,0,1,float16,fp8,0,0.7741653124491373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,64,0,1,fp8,fp8,0,0.7228319644927979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,64,128,1,float16,fp8,0,0.4501173496246338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,64,0,1,float16,float16,0,0.7804693380991617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,64,128,1,fp8,fp8,0,0.5775253375371298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,64,0,1,float16,fp8,0,0.7760907014211019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,64,128,1,float16,float16,0,0.3255733251571655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,64,0,1,fp8,fp8,0,0.7244799931844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,64,0,1,float16,float16,0,0.489898681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,64,128,1,float16,fp8,0,0.3245120048522949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,64,128,1,float16,fp8,0,0.3230239947636922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,64,128,1,fp8,fp8,0,0.39002664883931476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,64,0,1,float16,fp8,0,0.49109868208567303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,64,0,1,fp8,fp8,0,0.44729065895080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,64,128,1,float16,float16,0,0.3222080071767171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,64,0,1,float16,float16,0,0.4906986554463704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,64,128,1,fp8,fp8,0,0.38789331912994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,64,0,1,float16,fp8,0,0.49158398310343426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,64,0,1,fp8,fp8,0,0.44554134209950763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,64,0,1,fp8,fp8,0,0.4459253152211507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,64,128,1,float16,float16,0,0.32222400108973187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,64,0,1,float16,float16,0,0.4923199812571208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,64,128,1,float16,fp8,0,0.32291199763615924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,64,128,1,fp8,fp8,0,0.388810674349467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,64,128,1,fp8,fp8,0,0.3880266745885213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,64,0,1,float16,fp8,0,0.49141331513722736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,64,128,1,float16,float16,0,0.3230186700820923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,64,0,1,float16,float16,0,0.4930826822916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,64,128,1,float16,fp8,0,0.3230453332265218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,64,0,1,float16,float16,0,0.4912480115890503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,64,0,1,float16,fp8,0,0.49088533719380695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,64,0,1,fp8,fp8,0,0.44681068261464435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,64,128,1,float16,float16,0,0.3233013351758321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,64,128,1,float16,fp8,0,0.3230560024579366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,64,128,1,fp8,fp8,0,0.38805333773295086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,64,0,1,float16,fp8,0,0.49219199021657306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,64,0,1,fp8,fp8,0,0.4458453257878621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,64,128,1,float16,float16,0,8.41046396891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,64,128,1,float16,fp8,0,8.288415908813477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,64,128,1,float16,float16,0,8.37829844156901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,64,128,1,fp8,fp8,0,10.275866826375326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,64,0,1,float16,float16,0,12.104320526123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,64,0,1,fp8,fp8,0,10.594095865885416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,64,0,1,float16,fp8,0,12.004875183105469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,64,0,1,float16,float16,0,12.10546620686849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,64,128,1,float16,fp8,0,8.221216201782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,64,128,1,fp8,fp8,0,10.328453063964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,64,128,1,float16,float16,0,8.316933314005533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,64,128,1,float16,fp8,0,8.233029047648111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,64,0,1,fp8,fp8,0,10.842234293619791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,64,0,1,float16,fp8,0,12.01315689086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,64,0,1,float16,float16,0,12.126820882161459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,64,128,1,fp8,fp8,0,10.55349858601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,64,128,1,float16,float16,0,8.388325373331705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,64,0,1,fp8,fp8,0,10.865578969319662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,64,0,1,float16,fp8,0,12.075008392333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,64,128,1,float16,fp8,0,8.35099728902181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,64,128,1,fp8,fp8,0,10.519477208455404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,64,0,1,float16,float16,0,12.344341278076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,64,128,1,float16,float16,0,4.88043212890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,64,128,1,float16,fp8,0,4.806874593098958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,64,0,1,fp8,fp8,0,11.069296518961588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,64,0,1,float16,fp8,0,12.687615712483725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,64,0,1,float16,float16,0,6.800256093343099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,64,128,1,fp8,fp8,0,6.066783905029297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,64,128,1,float16,float16,0,4.015349388122559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,64,0,1,float16,fp8,0,6.723349253336589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,64,128,1,float16,fp8,0,3.9842453002929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,64,0,1,fp8,fp8,0,6.233866373697917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,64,0,1,float16,float16,0,6.064373016357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,64,128,1,fp8,fp8,0,5.154341379801433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,64,0,1,float16,fp8,0,5.850261052449544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,64,128,1,float16,float16,0,4.119301478068034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,64,0,1,fp8,fp8,0,5.264186541239421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,64,0,1,float16,float16,0,5.982485453287761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,64,128,1,float16,fp8,0,4.041573206583659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,64,128,1,fp8,fp8,0,5.215930620829265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,64,0,1,float16,fp8,0,5.9521331787109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,64,0,1,fp8,fp8,0,5.402410507202148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,64,128,1,float16,float16,0,4.127423922220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,64,128,1,float16,fp8,0,4.083466529846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,64,128,1,fp8,fp8,0,5.159008026123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,64,0,1,float16,float16,0,6.004079818725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,64,128,1,float16,float16,0,4.155061403910319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,64,0,1,float16,fp8,0,5.971157073974609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,64,0,1,fp8,fp8,0,5.37881596883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,64,128,1,float16,fp8,0,4.133914629618327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,64,0,1,float16,float16,0,6.032186508178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,64,128,1,fp8,fp8,0,5.163488070170085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,64,128,1,float16,float16,0,2.425701300303141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,64,0,1,float16,float16,0,3.3704748153686523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,64,128,1,float16,fp8,0,2.369738737742106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,64,0,1,fp8,fp8,0,5.381365458170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,64,0,1,float16,fp8,0,6.001717249552409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,64,128,1,fp8,fp8,0,2.995936075846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,64,0,1,float16,fp8,0,3.387493451436361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,64,0,1,fp8,fp8,0,3.078789393107096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,64,128,1,float16,float16,0,2.0102666219075522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,64,0,1,float16,float16,0,2.947429339090983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,64,128,1,float16,fp8,0,1.9850346247355144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,64,128,1,fp8,fp8,0,2.5669652620951333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,64,0,1,float16,fp8,0,2.9135093688964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,64,0,1,fp8,fp8,0,2.6639466285705566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,64,128,1,float16,float16,0,2.0320053100585938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,64,128,1,float16,fp8,0,1.9937866528828938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,64,0,1,float16,float16,0,2.9484640757242837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,64,128,1,fp8,fp8,0,2.565989335378011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,64,128,1,float16,float16,0,2.0129706064860025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,64,0,1,fp8,fp8,0,2.6605493227640786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,64,0,1,float16,fp8,0,2.9691041310628257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,64,0,1,float16,float16,0,2.9494880040486655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,64,128,1,float16,fp8,0,2.0085867245992026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,64,128,1,fp8,fp8,0,2.5746933619181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,64,0,1,fp8,fp8,0,2.648874600728353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,64,0,1,float16,fp8,0,2.9500745137532554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,64,128,1,float16,float16,0,2.021418730417887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,64,128,1,float16,fp8,0,2.016757329305013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,64,128,1,fp8,fp8,0,2.5834879875183105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,64,0,1,float16,float16,0,2.9837331771850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,64,0,1,float16,fp8,0,2.9473546346028647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,64,128,1,float16,float16,0,1.2177333037058513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,64,0,1,float16,float16,0,1.6858080228169758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,64,128,1,float16,fp8,0,1.191648006439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,64,0,1,fp8,fp8,0,2.6601120630900064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,64,128,1,fp8,fp8,0,1.46506134668986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,64,0,1,float16,fp8,0,1.6609973907470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,64,0,1,fp8,fp8,0,1.5162506103515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,64,128,1,fp8,fp8,0,1.324837366739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,64,128,1,float16,float16,0,1.034885327021281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,64,0,1,float16,fp8,0,1.469871997833252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,64,128,1,float16,fp8,0,1.0232213338216145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,64,0,1,float16,float16,0,1.507578690846761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,64,0,1,fp8,fp8,0,1.3650933901468914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,64,128,1,float16,float16,0,1.0429173310597737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,64,128,1,float16,fp8,0,1.0257439613342285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,64,0,1,float16,float16,0,1.5118613243103027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,64,128,1,fp8,fp8,0,1.3290826479593914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,64,128,1,float16,float16,0,1.0444533030192058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,64,0,1,float16,fp8,0,1.4731094042460124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,64,0,1,fp8,fp8,0,1.3747785886128743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,64,128,1,float16,fp8,0,1.0342293580373128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,64,0,1,float16,float16,0,1.5129013061523438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,64,128,1,fp8,fp8,0,1.326522668202718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,64,128,1,float16,float16,0,1.0399999618530273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,64,0,1,float16,fp8,0,1.5005280176798503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,64,0,1,fp8,fp8,0,1.3763093948364258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,64,128,1,float16,fp8,0,1.035306692123413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,64,0,1,float16,float16,0,1.50654935836792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,64,128,1,fp8,fp8,0,1.329914649327596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,64,128,1,float16,float16,0,0.6260960102081299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,64,0,1,fp8,fp8,0,1.3695519765218098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,64,0,1,float16,fp8,0,1.4928693771362305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,64,0,1,float16,float16,0,0.8658986886342367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,64,128,1,float16,fp8,0,0.6140373150507609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,64,128,1,fp8,fp8,0,0.7766613165537516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,64,0,1,float16,fp8,0,0.8495413462320963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,64,0,1,fp8,fp8,0,0.7882026831309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,64,128,1,float16,float16,0,0.5494453509648641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,64,0,1,float16,float16,0,0.7809120019276937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,64,0,1,float16,float16,0,0.7906560103098551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,64,128,1,float16,fp8,0,0.5445599953333536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,64,128,1,float16,fp8,0,0.537722667058309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,64,128,1,fp8,fp8,0,0.7018773555755615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,64,0,1,float16,fp8,0,0.7781759897867838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,64,0,1,fp8,fp8,0,0.7214773495992025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,64,128,1,float16,float16,0,0.5512319803237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,64,128,1,fp8,fp8,0,0.7055306434631348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,64,0,1,float16,fp8,0,0.7836266358693441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,64,0,1,fp8,fp8,0,0.7267359892527262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,64,128,1,float16,float16,0,0.5571253299713135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,64,0,1,float16,float16,0,0.7929386297861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,64,0,1,fp8,fp8,0,0.7279520034790039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,64,128,1,float16,fp8,0,0.5485599835713705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,64,128,1,fp8,fp8,0,0.7085493405659994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,64,0,1,float16,fp8,0,0.7836373647054037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,64,128,1,fp8,fp8,0,0.7098026275634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,64,128,1,float16,float16,0,0.5627573331197103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,64,0,1,float16,float16,0,0.7958079973856608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,64,128,1,float16,fp8,0,0.5506453514099121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,64,0,1,float16,fp8,0,0.7888267040252686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,64,0,1,fp8,fp8,0,0.7298879623413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,64,128,1,float16,float16,0,0.3380320072174072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,64,0,1,float16,float16,0,0.45818134148915607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,64,128,1,float16,fp8,0,0.34003734588623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,64,128,1,float16,fp8,0,0.3051360050837199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,64,128,1,fp8,fp8,0,0.42981334527333576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,64,0,1,float16,fp8,0,0.4591519832611084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,64,0,1,fp8,fp8,0,0.4294666846593221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,64,128,1,float16,float16,0,0.31035733222961426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,64,0,1,float16,float16,0,0.43115735054016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,64,0,1,float16,float16,0,0.4306666851043701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,64,128,1,fp8,fp8,0,0.3931359847386678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,64,0,1,fp8,fp8,0,0.3890506823857625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,64,0,1,float16,fp8,0,0.4272266626358032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,64,128,1,float16,float16,0,0.31093867619832355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,64,128,1,float16,fp8,0,0.30712000528971356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,64,128,1,fp8,fp8,0,0.3941919803619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,64,0,1,float16,fp8,0,0.4253600041071574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,64,0,1,fp8,fp8,0,0.3903253475824992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,64,128,1,float16,float16,0,0.30883200963338214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,64,0,1,float16,float16,0,0.4310773213704427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,64,128,1,float16,fp8,0,0.30828799804051715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,64,128,1,fp8,fp8,0,0.3944053252538045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,64,0,1,float16,fp8,0,0.42789332071940106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,64,0,1,fp8,fp8,0,0.38944534460703534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,64,128,1,float16,float16,0,0.3110293348630269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,64,0,1,float16,float16,0,0.4329440196355184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,64,128,1,float16,fp8,0,0.30848000446955365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,64,128,1,fp8,fp8,0,0.3962826728820801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,64,0,1,float16,fp8,0,0.43030933539072674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,64,0,1,fp8,fp8,0,0.39350934823354083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,64,128,1,float16,float16,0,0.2253119945526123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,64,0,1,float16,float16,0,0.2908693353335063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,64,128,1,float16,fp8,0,0.2258666753768921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,64,128,1,fp8,fp8,0,0.2720959981282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,64,0,1,float16,fp8,0,0.2906559904416402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,64,0,1,fp8,fp8,0,0.26128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,64,128,1,float16,float16,0,0.2224959929784139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,64,0,1,float16,float16,0,0.27426666021347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,64,128,1,float16,fp8,0,0.22336532672246298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,64,128,1,fp8,fp8,0,0.2698026696840922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,64,0,1,float16,fp8,0,0.2739413380622864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,64,0,1,fp8,fp8,0,0.2595679958661397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,64,128,1,float16,float16,0,0.2233546574910482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,64,0,1,float16,float16,0,0.27432533105214435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,64,128,1,float16,fp8,0,0.22245333592096964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,64,128,1,fp8,fp8,0,0.2701333363850911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,64,0,1,float16,fp8,0,0.27448532978693646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,64,0,1,fp8,fp8,0,0.25944532950719196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,64,128,1,float16,float16,0,0.2226933240890503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,64,0,1,float16,float16,0,0.27409066756566364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,64,128,1,float16,fp8,0,0.22322134176890054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,64,128,1,fp8,fp8,0,0.2702133258183797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,64,0,1,float16,fp8,0,0.2744906743367513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,64,0,1,fp8,fp8,0,0.2598293423652649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,64,128,1,float16,float16,0,0.22330133120218912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,64,0,1,float16,float16,0,0.274234672387441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,64,128,1,float16,fp8,0,0.2235893408457438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,64,128,1,fp8,fp8,0,0.27106134096781415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,64,0,1,float16,fp8,0,0.2741760015487671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,64,0,1,fp8,fp8,0,0.2595946590105693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,64,128,1,float16,float16,0,6.189119974772136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,64,128,1,float16,fp8,0,6.129248301188151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,64,128,1,float16,float16,0,6.106367746988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,64,0,1,fp8,fp8,0,6.908053080240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,64,0,1,float16,float16,0,7.926005045572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,64,128,1,fp8,fp8,0,7.626010894775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,64,0,1,float16,fp8,0,7.824442545572917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,64,0,1,float16,float16,0,7.883344014485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,64,128,1,float16,fp8,0,6.0281117757161455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,64,128,1,fp8,fp8,0,7.5927734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,64,128,1,float16,float16,0,6.09768549601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,64,0,1,fp8,fp8,0,6.943397521972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,64,0,1,float16,fp8,0,7.798117319742839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,64,128,1,float16,fp8,0,6.005786895751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,64,0,1,float16,float16,0,7.860975901285808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,64,128,1,fp8,fp8,0,7.6968638102213545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,64,0,1,float16,fp8,0,7.826229095458984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,64,128,1,float16,float16,0,6.130149205525716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,64,128,1,float16,fp8,0,6.0451094309488935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,64,0,1,fp8,fp8,0,6.970186869303386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,64,0,1,float16,float16,0,7.928234736124675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,64,128,1,fp8,fp8,0,7.720197041829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,64,128,1,float16,float16,0,3.6204214096069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,64,0,1,float16,fp8,0,7.862368265787761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,64,0,1,fp8,fp8,0,7.089429219563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,64,128,1,float16,fp8,0,3.5405972798665366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,64,0,1,float16,float16,0,4.507439931233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,64,0,1,float16,fp8,0,4.453813234965007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,64,128,1,fp8,fp8,0,4.489530563354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,64,128,1,float16,float16,0,2.938528060913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,64,128,1,float16,fp8,0,2.9003947575887046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,64,0,1,fp8,fp8,0,4.1677811940511065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,64,0,1,float16,float16,0,3.869317372639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,64,128,1,fp8,fp8,0,3.852810541788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,64,0,1,fp8,fp8,0,3.3998613357543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,64,128,1,float16,float16,0,2.977562586466471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,64,0,1,float16,fp8,0,3.8158400853474936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,64,128,1,float16,fp8,0,2.9663146336873374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,64,0,1,float16,float16,0,3.891354560852051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,64,128,1,fp8,fp8,0,3.812821388244629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,64,0,1,float16,fp8,0,3.861626625061035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,64,128,1,float16,float16,0,2.9906880060831704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,64,0,1,fp8,fp8,0,3.4977601369222007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,64,128,1,float16,fp8,0,2.9639358520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,64,0,1,float16,float16,0,3.9057706197102866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,64,128,1,fp8,fp8,0,3.820656140645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,64,0,1,float16,fp8,0,3.885893185933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,64,0,1,fp8,fp8,0,3.4761120478312173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,64,128,1,float16,float16,0,3.029002825419108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,64,128,1,float16,fp8,0,3.0210933685302734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,64,0,1,float16,float16,0,3.928192138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,64,128,1,float16,float16,0,1.7991520563761394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,64,128,1,fp8,fp8,0,3.8255093892415366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,64,0,1,fp8,fp8,0,3.505162556966146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,64,0,1,float16,fp8,0,3.900202751159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,64,0,1,float16,float16,0,2.2386560440063477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,64,128,1,float16,fp8,0,1.7463092803955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,64,128,1,fp8,fp8,0,2.162245273590088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,64,0,1,float16,fp8,0,2.1884640057881675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,64,128,1,float16,float16,0,1.4901280403137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,64,0,1,fp8,fp8,0,1.9846879641215007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,64,0,1,float16,float16,0,1.9299306869506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,64,128,1,float16,fp8,0,1.470128059387207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,64,128,1,fp8,fp8,0,1.9055946667989094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,64,0,1,float16,fp8,0,1.905461311340332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,64,0,1,fp8,fp8,0,1.7247573534647624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,64,128,1,float16,float16,0,1.5021066665649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,64,128,1,float16,fp8,0,1.4775360425313313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,64,0,1,float16,float16,0,1.9329919815063477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,64,128,1,fp8,fp8,0,1.9102773666381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,64,0,1,fp8,fp8,0,1.735103925069173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,64,0,1,float16,fp8,0,1.9162559509277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,64,128,1,float16,float16,0,1.4986720085144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,64,128,1,float16,fp8,0,1.4795519510904949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,64,0,1,float16,float16,0,1.9374079704284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,64,128,1,fp8,fp8,0,1.9203039805094402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,64,0,1,float16,fp8,0,1.9049386978149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,64,128,1,float16,float16,0,1.509402592976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,64,0,1,fp8,fp8,0,1.738250732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,64,128,1,float16,fp8,0,1.4871679941813152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,64,0,1,float16,float16,0,1.9473759333292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,64,128,1,fp8,fp8,0,1.9247466723124187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,64,0,1,float16,fp8,0,1.9153226216634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,64,128,1,float16,float16,0,0.9066932996114095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,64,0,1,fp8,fp8,0,1.7258826891581218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,64,128,1,float16,fp8,0,0.8906293710072836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,64,0,1,float16,float16,0,1.1349759896596272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,64,128,1,fp8,fp8,0,1.0984000364939372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,64,0,1,float16,fp8,0,1.1123039722442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,64,128,1,float16,float16,0,0.7720426718393961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,64,0,1,fp8,fp8,0,1.0051733652750652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,64,0,1,float16,float16,0,0.9860266844431559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,64,128,1,float16,fp8,0,0.7513919671376547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,64,0,1,fp8,fp8,0,0.8924000263214111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,64,128,1,fp8,fp8,0,0.9858826796213785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,64,128,1,float16,float16,0,0.7757279872894287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,64,0,1,float16,fp8,0,0.9826560020446777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,64,128,1,float16,fp8,0,0.7659467061360677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,64,0,1,float16,float16,0,0.9945013523101807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,64,128,1,fp8,fp8,0,0.9827520052591959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,64,0,1,float16,fp8,0,0.984608014424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,64,0,1,fp8,fp8,0,0.8965439796447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,64,128,1,float16,float16,0,0.7771413326263428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,64,128,1,float16,fp8,0,0.7669760386149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,64,128,1,float16,float16,0,0.7810880343119303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,64,0,1,float16,float16,0,1.0038879712422688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,64,128,1,fp8,fp8,0,0.9909599622090658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,64,0,1,float16,fp8,0,0.9847306410471598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,64,0,1,fp8,fp8,0,0.8924372990926107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,64,128,1,float16,fp8,0,0.7701120376586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,64,0,1,float16,float16,0,0.9986773331960043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,64,128,1,fp8,fp8,0,1.0028213659922283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,64,0,1,float16,fp8,0,0.9921173254648844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,64,128,1,float16,float16,0,0.4734079837799072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,64,0,1,fp8,fp8,0,0.9083200295766195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,64,128,1,float16,fp8,0,0.4652533531188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,64,0,1,float16,float16,0,0.5852053165435791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,64,128,1,fp8,fp8,0,0.5857546726862589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,64,0,1,float16,fp8,0,0.5792853434880575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,64,0,1,fp8,fp8,0,0.5338826576868693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,64,128,1,float16,float16,0,0.4129120111465454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,64,0,1,float16,float16,0,0.5293279886245728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,64,128,1,float16,fp8,0,0.4097226858139038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,64,128,1,fp8,fp8,0,0.5320746501286825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,64,0,1,float16,fp8,0,0.5229333241780599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,64,128,1,fp8,fp8,0,0.532314658164978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,64,0,1,fp8,fp8,0,0.4792426824569702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,64,128,1,float16,float16,0,0.41969064871470135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,64,128,1,float16,float16,0,0.4193333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,64,0,1,float16,float16,0,0.5318613449732462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,64,128,1,float16,fp8,0,0.41282665729522705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,64,0,1,float16,fp8,0,0.5244853496551514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,64,0,1,fp8,fp8,0,0.4840159813563029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,64,0,1,float16,float16,0,0.5305973291397095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,64,128,1,fp8,fp8,0,0.5355573495229086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,64,128,1,float16,fp8,0,0.41493332386016846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,64,0,1,float16,fp8,0,0.5275839964548746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,64,0,1,fp8,fp8,0,0.4840693473815918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,64,128,1,float16,float16,0,0.42127466201782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,64,0,1,float16,float16,0,0.5342986583709717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,64,128,1,float16,fp8,0,0.4149920145670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,64,128,1,fp8,fp8,0,0.5372533400853475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,64,0,1,float16,fp8,0,0.5296159982681274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,64,0,1,fp8,fp8,0,0.48868799209594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,64,128,1,float16,float16,0,0.25939732789993286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,64,0,1,float16,float16,0,0.315775990486145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,64,128,1,float16,fp8,0,0.26016533374786377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,64,0,1,float16,fp8,0,0.31546666224797565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,64,128,1,fp8,fp8,0,0.3290826678276062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,64,0,1,fp8,fp8,0,0.2927626570065816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,64,128,1,float16,float16,0,0.23768534262975058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,64,0,1,float16,float16,0,0.29502399762471515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,64,128,1,float16,fp8,0,0.23475199937820435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,64,128,1,fp8,fp8,0,0.301962673664093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,64,0,1,float16,fp8,0,0.2902453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,64,0,1,fp8,fp8,0,0.266975998878479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,64,128,1,float16,float16,0,0.23668799797693887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,64,0,1,float16,float16,0,0.29546666145324707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,64,128,1,float16,fp8,0,0.23414399226506552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,64,128,1,fp8,fp8,0,0.30421332518259686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,64,0,1,float16,fp8,0,0.291978657245636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,64,0,1,float16,fp8,0,0.2930720051129659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,64,0,1,fp8,fp8,0,0.2661813298861186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,64,0,1,fp8,fp8,0,0.26476265986760456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,64,128,1,float16,float16,0,0.23779199520746866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,64,0,1,float16,float16,0,0.2984480063120524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,64,128,1,float16,fp8,0,0.23463465770085654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,64,128,1,fp8,fp8,0,0.3046026627222697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,64,128,1,float16,float16,0,0.23885866006215414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,64,0,1,float16,float16,0,0.29799999793370563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,64,0,1,float16,float16,0,0.20046399037043253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,64,128,1,float16,fp8,0,0.23760533332824707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,64,128,1,fp8,fp8,0,0.30478399991989136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,64,0,1,float16,fp8,0,0.2945599953333537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,64,0,1,fp8,fp8,0,0.2675466736157735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,64,128,1,float16,float16,0,0.17432000239690146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,64,128,1,float16,fp8,0,0.17529600858688354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,64,128,1,fp8,fp8,0,0.21276267369588217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,64,0,1,float16,fp8,0,0.2005386749903361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,64,0,1,float16,fp8,0,0.19210133949915567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,64,0,1,fp8,fp8,0,0.18316266934076944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,64,0,1,fp8,fp8,0,0.18413333098093668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,64,128,1,float16,float16,0,0.1723466714223226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,64,0,1,float16,float16,0,0.1919999917348226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,64,128,1,float16,fp8,0,0.17195733388264975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,64,128,1,fp8,fp8,0,0.20954134066899618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,64,128,1,float16,float16,0,0.17245866854985556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,64,0,1,float16,float16,0,0.19189866383870444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,64,128,1,float16,fp8,0,0.17292267084121704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,64,128,1,float16,fp8,0,0.17419199148813883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,64,128,1,fp8,fp8,0,0.21092800299326578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,64,0,1,float16,fp8,0,0.19274665911992392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,64,0,1,fp8,fp8,0,0.1834026575088501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,64,128,1,float16,float16,0,0.1726400057474772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,64,0,1,float16,float16,0,0.1925013264020284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,64,0,1,float16,float16,0,0.19178134202957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,64,128,1,fp8,fp8,0,0.2102880080540975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,64,0,1,float16,fp8,0,0.193615992863973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,64,0,1,fp8,fp8,0,0.18361600240071616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,64,128,1,float16,float16,0,0.17298134167989096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,64,128,1,float16,fp8,0,0.17334934075673422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,64,128,1,fp8,fp8,0,0.2112213373184204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,64,0,1,float16,fp8,0,0.19266132513682047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,64,0,1,fp8,fp8,0,0.18336532513300577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,64,128,1,float16,float16,0,7.217536290486653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,64,0,1,float16,float16,0,8.382778803507486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,64,128,1,float16,fp8,0,7.219290415445964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,64,0,1,float16,fp8,0,8.386677424112955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,64,0,1,fp8,fp8,0,7.294464111328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,64,128,1,fp8,fp8,0,9.306874593098959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,64,128,1,float16,float16,0,7.343242645263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,64,128,1,float16,fp8,0,7.248965581258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,64,0,1,float16,float16,0,8.387615839640299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,64,128,1,fp8,fp8,0,9.437909444173178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,64,0,1,fp8,fp8,0,7.292842864990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,64,0,1,float16,fp8,0,8.398858388264975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,64,128,1,float16,float16,0,7.3946183522542315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,64,128,1,float16,fp8,0,7.408112208048503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,64,0,1,float16,float16,0,8.523712158203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,64,0,1,float16,fp8,0,8.499802907307943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,64,128,1,fp8,fp8,0,9.421157201131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,64,0,1,fp8,fp8,0,7.378586451212565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,64,128,1,float16,float16,0,7.489765167236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,64,128,1,float16,fp8,0,7.536549250284831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,64,0,1,float16,float16,0,8.629877090454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,64,0,1,fp8,fp8,0,7.440842946370442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,64,128,1,float16,float16,0,4.526639938354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,64,128,1,fp8,fp8,0,9.53595225016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,64,0,1,float16,fp8,0,8.647221247355143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,64,0,1,float16,float16,0,5.0879465738932295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,64,128,1,float16,fp8,0,4.421360015869141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,64,0,1,float16,fp8,0,4.93120543162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,64,128,1,fp8,fp8,0,5.258901278177897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,64,128,1,float16,float16,0,3.563349405924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,64,0,1,fp8,fp8,0,4.231103897094727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,64,128,1,float16,fp8,0,3.5883200963338218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,64,0,1,float16,float16,0,4.136234601338704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,64,0,1,fp8,fp8,0,3.5938453674316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,64,0,1,float16,fp8,0,4.182959874471028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,64,128,1,fp8,fp8,0,4.630613327026367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,64,128,1,float16,float16,0,3.6068747838338218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,64,128,1,float16,fp8,0,3.5851465861002603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,64,0,1,float16,float16,0,4.182245254516602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,64,128,1,fp8,fp8,0,4.651893297831218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,64,0,1,float16,fp8,0,4.168154716491699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,64,0,1,fp8,fp8,0,3.651930809020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,64,128,1,float16,fp8,0,3.696634610493978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,64,128,1,float16,float16,0,3.6838293075561523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,64,0,1,float16,float16,0,4.19981320699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,64,128,1,fp8,fp8,0,4.695823987325032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,64,128,1,float16,float16,0,3.7916587193806968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,64,0,1,float16,fp8,0,4.2762454350789385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,64,0,1,fp8,fp8,0,3.703477223714193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,64,128,1,float16,fp8,0,3.7932799657185874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,64,0,1,float16,float16,0,4.374746640523274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,64,128,1,fp8,fp8,0,4.819989204406738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,64,128,1,float16,float16,0,2.2672053972880044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,64,0,1,float16,fp8,0,4.337039947509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,64,0,1,fp8,fp8,0,3.7547518412272134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,64,0,1,float16,float16,0,2.5619680086771646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,64,128,1,float16,fp8,0,2.2020586331685386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,64,128,1,float16,float16,0,1.7893493970235188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,64,128,1,fp8,fp8,0,2.6385706265767417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,64,0,1,float16,fp8,0,2.4893919626871743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,64,0,1,fp8,fp8,0,2.1051732699076333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,64,0,1,float16,float16,0,2.0745066006978354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,64,128,1,float16,fp8,0,1.789632002512614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,64,128,1,fp8,fp8,0,2.3889333407084146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,64,0,1,float16,fp8,0,2.095370610555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,64,0,1,fp8,fp8,0,1.809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,64,128,1,float16,float16,0,1.8115839958190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,64,0,1,float16,float16,0,2.1143946647644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,64,0,1,fp8,fp8,0,1.8400373458862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,64,128,1,float16,float16,0,1.8304959932963054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,64,128,1,float16,fp8,0,1.8487626711527507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,64,128,1,fp8,fp8,0,2.352351983388265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,64,0,1,float16,fp8,0,2.106485366821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,64,0,1,float16,float16,0,2.1214292844136557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,64,128,1,float16,fp8,0,1.8305706977844238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,64,128,1,fp8,fp8,0,2.4029599825541177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,64,0,1,float16,fp8,0,2.1163786252339682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,64,0,1,fp8,fp8,0,1.8540266354878743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,64,128,1,float16,fp8,0,1.8837812741597493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,64,128,1,float16,float16,0,1.8994773228963215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,64,128,1,fp8,fp8,0,2.4143999417622886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,64,0,1,float16,float16,0,2.17576535542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,64,0,1,float16,fp8,0,2.176191965738932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,64,128,1,float16,float16,0,1.1442773342132568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,64,0,1,fp8,fp8,0,1.8882880210876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,64,0,1,float16,float16,0,1.279194672902425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,64,0,1,fp8,fp8,0,1.0588693618774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,64,128,1,float16,fp8,0,1.1103359858194988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,64,128,1,fp8,fp8,0,1.3324693044026692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,64,0,1,float16,fp8,0,1.2414666811625164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,64,128,1,float16,float16,0,0.9174239635467529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,64,0,1,float16,float16,0,1.060698668162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,64,128,1,float16,fp8,0,0.9222026666005453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,64,128,1,fp8,fp8,0,1.1981173356374104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,64,0,1,float16,fp8,0,1.0709013144175212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,64,0,1,fp8,fp8,0,0.9277919928232828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,64,0,1,float16,fp8,0,1.074938694636027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,64,128,1,float16,float16,0,0.928394635518392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,64,128,1,float16,fp8,0,0.9308053652445475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,64,0,1,float16,float16,0,1.0679787000020344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,64,128,1,fp8,fp8,0,1.20632537206014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,64,0,1,fp8,fp8,0,0.9303359985351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,64,0,1,fp8,fp8,0,0.9383467038472494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,64,128,1,float16,float16,0,0.9351466496785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,64,0,1,float16,float16,0,1.0738186836242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,64,128,1,float16,fp8,0,0.9299039840698242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,64,128,1,fp8,fp8,0,1.2098613580067952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,64,0,1,float16,fp8,0,1.0750079949696858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,64,128,1,float16,float16,0,0.9461332956949869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,64,0,1,float16,float16,0,1.0987892945607503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,64,128,1,float16,fp8,0,0.9508746465047201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,64,128,1,fp8,fp8,0,1.242186705271403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,64,0,1,float16,fp8,0,1.0959253311157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,64,0,1,fp8,fp8,0,0.9583946863810221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,64,128,1,float16,float16,0,0.5805866718292236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,64,0,1,float16,float16,0,0.653434673945109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,64,128,1,float16,fp8,0,0.565226674079895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,64,128,1,fp8,fp8,0,0.6827999750773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,64,0,1,float16,fp8,0,0.6369973421096802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,64,0,1,fp8,fp8,0,0.5407626628875732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,64,128,1,float16,float16,0,0.4765919844309489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,64,0,1,float16,float16,0,0.5513706604639689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,64,128,1,float16,float16,0,0.4806400140126546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,64,128,1,float16,fp8,0,0.4782506624857585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,64,0,1,float16,float16,0,0.5570293267567953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,64,128,1,float16,fp8,0,0.4809120098749797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,64,128,1,fp8,fp8,0,0.62281600634257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,64,0,1,float16,fp8,0,0.550869345664978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,64,0,1,fp8,fp8,0,0.47935465971628827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,64,128,1,fp8,fp8,0,0.62281600634257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,64,128,1,fp8,fp8,0,0.625434676806132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,64,0,1,fp8,fp8,0,0.48317865530649823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,64,0,1,float16,fp8,0,0.5522346496582031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,64,128,1,float16,float16,0,0.4805440107981364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,64,0,1,float16,float16,0,0.5588586727778116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,64,0,1,float16,float16,0,0.5523360172907511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,64,128,1,float16,fp8,0,0.4824639956156413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,64,0,1,float16,fp8,0,0.5578133265177408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,64,0,1,float16,fp8,0,0.553114652633667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,64,0,1,fp8,fp8,0,0.48370134830474854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,64,0,1,float16,float16,0,0.3383520046869914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,64,128,1,float16,float16,0,0.4856799840927124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,64,128,1,float16,fp8,0,0.48473600546518963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,64,128,1,fp8,fp8,0,0.6289120117823283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,64,128,1,float16,float16,0,0.25218133131663006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,64,0,1,fp8,fp8,0,0.4846133391062419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,64,0,1,float16,float16,0,0.29054399331410724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,64,128,1,float16,fp8,0,0.2542346715927124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,64,128,1,fp8,fp8,0,0.32949332396189374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,64,128,1,float16,float16,0,0.30275734265645343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,64,128,1,float16,fp8,0,0.2965173323949178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,64,128,1,fp8,fp8,0,0.361194650332133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,64,0,1,float16,fp8,0,0.330677330493927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,64,0,1,fp8,fp8,0,0.28222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,64,0,1,float16,fp8,0,0.2911466757456462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,64,0,1,fp8,fp8,0,0.24856533606847128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,64,128,1,float16,float16,0,0.2537173430124919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,64,0,1,float16,float16,0,0.29131199916203815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,64,128,1,float16,fp8,0,0.2542506655057271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,64,128,1,fp8,fp8,0,0.33219732840855914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,64,0,1,float16,fp8,0,0.2906559904416402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,64,0,1,fp8,fp8,0,0.2500800093015035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,64,128,1,float16,float16,0,0.25552000602086383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,64,0,1,float16,float16,0,0.2916586597760518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,64,128,1,float16,fp8,0,0.2564479907353719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,64,128,1,float16,fp8,0,0.2563093304634094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,64,128,1,fp8,fp8,0,0.33208000659942627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,64,0,1,float16,fp8,0,0.29364800453186035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,64,0,1,fp8,fp8,0,0.2518133322397868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,64,0,1,float16,fp8,0,0.2924906611442566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,64,0,1,float16,float16,0,0.1925920049349467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,64,128,1,float16,fp8,0,0.16180266936620077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,64,0,1,fp8,fp8,0,0.25014400482177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,64,128,1,float16,float16,0,0.2549706697463989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,64,0,1,float16,float16,0,0.2935573259989421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,64,128,1,fp8,fp8,0,0.33483731746673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,64,128,1,float16,float16,0,0.16487999757130942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,64,128,1,fp8,fp8,0,0.19928000370661417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,64,0,1,float16,fp8,0,0.18744534254074097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,64,0,1,fp8,fp8,0,0.1497599979241689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,64,128,1,float16,float16,0,0.1381119986375173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,64,0,1,float16,float16,0,0.15103466312090555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,64,128,1,float16,fp8,0,0.13929067055384317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,64,128,1,fp8,fp8,0,0.18383467197418213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,64,0,1,float16,fp8,0,0.1516213317712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,64,0,1,fp8,fp8,0,0.13447999954223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,64,128,1,float16,float16,0,0.13903466860453287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,64,0,1,float16,float16,0,0.151936004559199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,64,128,1,float16,fp8,0,0.14004799723625183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,64,128,1,fp8,fp8,0,0.1844693422317505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,64,0,1,float16,fp8,0,0.1523413360118866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,64,0,1,float16,fp8,0,0.15229333440462747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,64,0,1,fp8,fp8,0,0.13595199584960938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,64,128,1,float16,float16,0,0.14130133390426636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,64,0,1,float16,float16,0,0.15237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,64,128,1,float16,fp8,0,0.14170666535695395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,64,128,1,fp8,fp8,0,0.18480000893274942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,64,0,1,fp8,fp8,0,0.1355413297812144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,64,128,1,float16,float16,0,0.09384000301361084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,64,128,1,float16,float16,0,0.14122133453687033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,64,0,1,float16,float16,0,0.15242666999499002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,64,128,1,float16,fp8,0,0.14079466462135315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,64,128,1,fp8,fp8,0,0.18568533658981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,64,0,1,float16,fp8,0,0.153221329053243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,64,0,1,fp8,fp8,0,0.1381333371003469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,64,0,1,float16,float16,0,0.09667733311653137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,64,128,1,float16,fp8,0,0.09341333309809367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,64,128,1,fp8,fp8,0,0.11890133221944173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,64,0,1,float16,fp8,0,0.09614933530489604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,64,0,1,float16,fp8,0,0.08372267087300618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,64,0,1,fp8,fp8,0,0.08592533071835835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,64,128,1,float16,float16,0,0.07902400195598602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,64,128,1,float16,fp8,0,0.08045866588751475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,64,0,1,float16,float16,0,0.08377066254615784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,64,128,1,float16,fp8,0,0.07931733131408691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,64,128,1,fp8,fp8,0,0.1027893324693044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,64,0,1,fp8,fp8,0,0.07578666508197784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,64,128,1,float16,float16,0,0.08025066554546356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,64,0,1,float16,float16,0,0.08430400490760803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,64,128,1,fp8,fp8,0,0.10302933057149251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,64,0,1,float16,fp8,0,0.08457600076993306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,64,0,1,fp8,fp8,0,0.07600000003973643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,64,128,1,float16,float16,0,0.07998399933179219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,64,0,1,float16,float16,0,0.08541333675384521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,64,128,1,float16,fp8,0,0.0798880010843277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,64,128,1,fp8,fp8,0,0.10414399703343709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,64,0,1,float16,fp8,0,0.08474133412043254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,64,0,1,fp8,fp8,0,0.0757173349459966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,64,128,1,float16,float16,0,0.08005333443482716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,64,0,1,float16,float16,0,0.08462933699289958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,64,128,1,float16,fp8,0,0.081194669008255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,64,128,1,fp8,fp8,0,0.10416533549626668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,64,0,1,float16,fp8,0,0.08556800087292989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,64,0,1,fp8,fp8,0,0.07624533275763194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,64,128,1,float16,float16,0,6.857765197753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,64,0,1,float16,float16,0,6.809328079223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,64,128,1,float16,fp8,0,6.803381601969401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,64,0,1,float16,fp8,0,6.815999984741211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,64,0,1,fp8,fp8,0,5.9134877522786455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,64,0,1,float16,float16,0,6.946816126505534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,64,128,1,float16,float16,0,6.958437601725261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,64,128,1,fp8,fp8,0,8.790858586629232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,64,128,1,float16,fp8,0,6.889088312784831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,64,0,1,fp8,fp8,0,5.907674789428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,64,0,1,float16,fp8,0,6.917978922526042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,64,128,1,fp8,fp8,0,8.861087799072266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,64,128,1,float16,float16,0,7.008805592854817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,64,0,1,float16,float16,0,6.947274525960286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,64,128,1,float16,fp8,0,6.958341598510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,64,128,1,fp8,fp8,0,8.881280263264975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,64,0,1,fp8,fp8,0,5.94923718770345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,64,0,1,float16,fp8,0,6.943445205688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,64,128,1,float16,float16,0,7.128997166951497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,64,0,1,float16,float16,0,7.1446882883707685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,64,128,1,float16,fp8,0,7.088538487752278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,64,0,1,float16,fp8,0,7.08296012878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,64,0,1,fp8,fp8,0,6.062367757161458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,64,128,1,float16,float16,0,4.308821360270183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,64,0,1,float16,float16,0,4.272538820902507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,64,128,1,fp8,fp8,0,9.013408025105795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,64,128,1,float16,fp8,0,4.1922454833984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,64,0,1,float16,fp8,0,4.15557320912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,64,128,1,fp8,fp8,0,5.009994824727376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,64,0,1,fp8,fp8,0,3.5192267100016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,64,128,1,float16,float16,0,3.3488054275512695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,64,0,1,float16,float16,0,3.3292531967163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,64,128,1,float16,fp8,0,3.3497066497802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,64,128,1,fp8,fp8,0,4.357103983561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,64,0,1,float16,fp8,0,3.3470986684163413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,64,0,1,fp8,fp8,0,2.872549374898275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,64,128,1,float16,float16,0,3.404186566670736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,64,0,1,float16,float16,0,3.3927253087361655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,64,128,1,float16,fp8,0,3.3900426228841147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,64,0,1,float16,fp8,0,3.3718560536702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,64,0,1,fp8,fp8,0,2.9255040486653647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,64,128,1,fp8,fp8,0,4.4094133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,64,128,1,float16,float16,0,3.4634294509887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,64,0,1,float16,float16,0,3.4163039525349936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,64,128,1,float16,fp8,0,3.486703872680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,64,0,1,float16,fp8,0,3.441120147705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,64,0,1,fp8,fp8,0,2.955514589945475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,64,128,1,fp8,fp8,0,4.439034779866536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,64,128,1,float16,float16,0,3.5318241119384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,64,0,1,float16,float16,0,3.5342772801717124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,64,128,1,float16,float16,0,2.1491573651631675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,64,128,1,float16,fp8,0,3.5738932291666665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,64,0,1,float16,fp8,0,3.542880058288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,64,0,1,fp8,fp8,0,3.0076001485188804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,64,0,1,float16,float16,0,2.138213316599528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,64,128,1,float16,fp8,0,2.084218660990397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,64,128,1,fp8,fp8,0,4.472965240478516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,64,128,1,fp8,fp8,0,2.4867893854777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,64,0,1,float16,fp8,0,2.073882738749186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,64,0,1,fp8,fp8,0,1.750831921895345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,64,128,1,float16,float16,0,1.6959999402364094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,64,0,1,float16,float16,0,1.6796959241231282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,64,128,1,float16,fp8,0,1.703722635904948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,64,0,1,float16,fp8,0,1.6945759455362956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,64,128,1,fp8,fp8,0,2.2201600074768066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,64,128,1,float16,float16,0,1.7151734034220378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,64,0,1,fp8,fp8,0,1.4454453786214192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,64,0,1,float16,float16,0,1.740261395772298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,64,128,1,float16,fp8,0,1.7112959225972493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,64,0,1,float16,fp8,0,1.7066133817036946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,64,128,1,fp8,fp8,0,2.2121814092000327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,64,0,1,fp8,fp8,0,1.4682879447937012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,64,128,1,float16,float16,0,1.7167147000630696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,64,0,1,float16,float16,0,1.7013120651245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,64,128,1,float16,fp8,0,1.7285653750101726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,64,0,1,float16,fp8,0,1.7042880058288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,64,128,1,fp8,fp8,0,2.2625813484191895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,64,0,1,fp8,fp8,0,1.4881226221720378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,64,128,1,float16,float16,0,1.7770506540934246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,64,0,1,float16,float16,0,1.7757333119710286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,64,128,1,float16,fp8,0,1.770917256673177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,64,0,1,float16,fp8,0,1.7729652722676594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,64,128,1,fp8,fp8,0,2.266021410624186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,64,128,1,float16,float16,0,1.0930986404418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,64,0,1,float16,float16,0,1.08133864402771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,64,128,1,float16,fp8,0,1.0545120239257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,64,0,1,fp8,fp8,0,1.5164426167805989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,64,128,1,fp8,fp8,0,1.2548586527506511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,64,0,1,float16,fp8,0,1.0424799919128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,64,0,1,fp8,fp8,0,0.8713706334431967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,64,0,1,float16,float16,0,0.860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,64,128,1,float16,float16,0,0.8729653358459473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,64,128,1,float16,fp8,0,0.8633440335591634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,64,0,1,float16,fp8,0,0.8576160271962484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,64,128,1,fp8,fp8,0,1.1242773532867432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,64,0,1,fp8,fp8,0,0.7406026522318522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,64,128,1,float16,float16,0,0.8768373330434164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,64,0,1,float16,float16,0,0.8638880252838135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,64,128,1,float16,fp8,0,0.8810933430989584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,64,128,1,fp8,fp8,0,1.1301813125610352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,64,0,1,float16,fp8,0,0.8629546960194906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,64,0,1,fp8,fp8,0,0.7464160124460856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,64,128,1,float16,float16,0,0.87663467725118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,64,0,1,float16,float16,0,0.86572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,64,128,1,float16,fp8,0,0.8778560161590576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,64,0,1,float16,fp8,0,0.8682186603546143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,64,0,1,float16,float16,0,0.8862666289011637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,64,128,1,fp8,fp8,0,1.1370026270548503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,64,0,1,fp8,fp8,0,0.7476320266723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,64,128,1,float16,float16,0,0.8907787005106608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,64,128,1,float16,fp8,0,0.9031999905904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,64,128,1,fp8,fp8,0,1.1543839772542317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,64,0,1,float16,fp8,0,0.8806880315144857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,64,128,1,fp8,fp8,0,0.6484693288803101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,64,0,1,float16,fp8,0,0.5309333403905233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,64,0,1,fp8,fp8,0,0.7655306657155355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,64,128,1,float16,float16,0,0.5567093292872111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,64,0,1,float16,float16,0,0.4461386601130168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,64,0,1,float16,float16,0,0.5448640187581381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,64,128,1,float16,fp8,0,0.5402186711629232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,64,0,1,fp8,fp8,0,0.4463626543680827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,64,128,1,float16,float16,0,0.4514933427174886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,64,128,1,float16,fp8,0,0.4524799982706706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,64,0,1,float16,float16,0,0.4479466676712036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,64,128,1,fp8,fp8,0,0.5829279820124308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,64,0,1,float16,fp8,0,0.44706666469573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,64,0,1,fp8,fp8,0,0.38491201400756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,64,128,1,float16,float16,0,0.4540426731109619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,64,128,1,float16,fp8,0,0.4548693497975667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,64,128,1,fp8,fp8,0,0.5859306653340658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,64,0,1,float16,fp8,0,0.44658132394154865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,64,0,1,fp8,fp8,0,0.38789331912994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,64,128,1,float16,float16,0,0.4554773171742757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,64,0,1,float16,float16,0,0.44836799303690594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,64,128,1,float16,fp8,0,0.4562079906463623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,64,128,1,fp8,fp8,0,0.5873653491338094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,64,0,1,float16,fp8,0,0.4479626814524333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,64,0,1,fp8,fp8,0,0.3873279889424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,64,128,1,float16,float16,0,0.4591999848683675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,64,0,1,float16,float16,0,0.4537866512934367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,64,128,1,float16,fp8,0,0.4589759906133016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,64,0,1,float16,fp8,0,0.4511733452479045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,64,128,1,fp8,fp8,0,0.5929493506749471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,64,0,1,fp8,fp8,0,0.39447466532389325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,64,128,1,float16,float16,0,0.2908160090446472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,64,0,1,float16,float16,0,0.28385066986083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,64,128,1,float16,fp8,0,0.28379732370376587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,64,128,1,fp8,fp8,0,0.3396426836649577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,64,0,1,float16,fp8,0,0.27749866247177124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,64,0,1,fp8,fp8,0,0.2327573299407959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,64,128,1,float16,float16,0,0.23985066016515097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,64,0,1,float16,float16,0,0.23413866758346558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,64,128,1,float16,fp8,0,0.24166399240493774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,64,128,1,fp8,fp8,0,0.31095999479293823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,64,0,1,float16,fp8,0,0.2358506719271342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,64,0,1,fp8,fp8,0,0.2041706641515096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,64,128,1,float16,float16,0,0.23996800184249878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,64,0,1,float16,float16,0,0.23575466871261597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,64,128,1,float16,fp8,0,0.24155733982721964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,64,128,1,fp8,fp8,0,0.31284799178441364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,64,0,1,float16,fp8,0,0.23686933517456055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,64,0,1,fp8,fp8,0,0.20410666863123575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,64,0,1,fp8,fp8,0,0.20771199464797974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,64,128,1,float16,float16,0,0.2433919906616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,64,0,1,float16,float16,0,0.2390986680984497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,64,0,1,float16,float16,0,0.23735467592875162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,64,128,1,float16,fp8,0,0.2419040004412333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,64,0,1,fp8,fp8,0,0.20683199167251587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,64,128,1,fp8,fp8,0,0.31225599845250446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,64,0,1,float16,float16,0,0.16644799709320068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,64,0,1,float16,fp8,0,0.2363306681315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,64,128,1,float16,float16,0,0.24741866191228232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,64,128,1,fp8,fp8,0,0.1883946657180786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,64,0,1,float16,fp8,0,0.15959466497103372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,64,128,1,float16,fp8,0,0.2448373238245646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,64,128,1,fp8,fp8,0,0.3151893417040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,64,0,1,float16,fp8,0,0.23893332481384277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,64,128,1,float16,float16,0,0.15802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,64,128,1,fp8,fp8,0,0.1726613243420919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,64,128,1,float16,fp8,0,0.15576533476511636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,64,0,1,fp8,fp8,0,0.1275040010611216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,64,128,1,float16,float16,0,0.13224533200263977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,64,0,1,float16,float16,0,0.12570133805274963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,64,128,1,float16,fp8,0,0.1323093374570211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,64,0,1,float16,fp8,0,0.12710400422414145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,64,128,1,fp8,fp8,0,0.17409600814183554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,64,0,1,fp8,fp8,0,0.11248000462849934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,64,128,1,float16,float16,0,0.1338933308919271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,64,0,1,float16,float16,0,0.12803733348846436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,64,128,1,float16,fp8,0,0.13451733191808066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,64,0,1,float16,fp8,0,0.12771733601888022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,64,0,1,fp8,fp8,0,0.11255466938018799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,64,128,1,float16,float16,0,0.13361600041389465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,64,0,1,float16,float16,0,0.12736533085505167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,64,128,1,float16,fp8,0,0.1336426635583242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,64,128,1,fp8,fp8,0,0.17436800400416055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,64,128,1,float16,fp8,0,0.1346560021241506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,64,0,1,float16,fp8,0,0.12863999605178833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,64,0,1,fp8,fp8,0,0.11429333686828613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,64,128,1,float16,float16,0,0.13453333576520285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,64,0,1,float16,float16,0,0.12783466776212057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,64,128,1,fp8,fp8,0,0.17564266920089722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,64,0,1,float16,fp8,0,0.12916800379753113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,64,0,1,fp8,fp8,0,0.11441066861152649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,64,0,1,fp8,fp8,0,0.07550399998823802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,64,128,1,float16,float16,0,0.09153067072232564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,64,0,1,float16,float16,0,0.08705600102742513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,64,128,1,float16,fp8,0,0.090037335952123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,64,128,1,fp8,fp8,0,0.11308800180753072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,64,0,1,float16,fp8,0,0.08443733056386311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,64,128,1,float16,float16,0,0.07604266703128815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,64,0,1,float16,float16,0,0.07145600020885468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,64,128,1,float16,fp8,0,0.0768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,64,128,1,fp8,fp8,0,0.09605333209037781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,64,0,1,fp8,fp8,0,0.0631573349237442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,64,0,1,float16,fp8,0,0.07099733253320058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,64,128,1,float16,float16,0,0.0767146646976471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,64,0,1,float16,float16,0,0.07180800040562947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,64,128,1,float16,fp8,0,0.07653866708278656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,64,128,1,fp8,fp8,0,0.09777067104975383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,64,128,1,fp8,fp8,0,0.09742400050163269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,64,0,1,float16,fp8,0,0.07251733541488647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,64,0,1,fp8,fp8,0,0.06401066482067108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,64,0,1,fp8,fp8,0,0.06364800035953522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,64,128,1,float16,float16,0,0.07677866518497467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,64,0,1,float16,float16,0,0.07182399928569794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,64,128,1,float16,fp8,0,0.07715733349323273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,64,0,1,float16,fp8,0,0.07167466481526692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,64,128,1,float16,float16,0,0.07894933223724365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,64,0,1,float16,float16,0,0.07269333302974701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,64,128,1,float16,fp8,0,0.07838400204976399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,64,128,1,fp8,fp8,0,0.09704533219337463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,64,128,1,fp8,fp8,0,0.06993600229422252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,64,0,1,float16,fp8,0,0.07258666555086772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,64,0,1,fp8,fp8,0,0.06473066906134288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,64,128,1,float16,float16,0,0.05515733361244202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,64,0,1,float16,float16,0,0.0507893313964208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,64,128,1,float16,fp8,0,0.0551093320051829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,64,0,1,float16,fp8,0,0.050053333242734276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,64,0,1,fp8,fp8,0,0.04684266448020935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,64,128,1,float16,float16,0,0.04957333207130432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,64,0,1,float16,float16,0,0.04574400186538696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,64,128,1,float16,fp8,0,0.05026133358478546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,64,128,1,fp8,fp8,0,0.06327466666698456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,64,0,1,float16,fp8,0,0.04604800045490265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,64,0,1,fp8,fp8,0,0.04195733368396759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,64,128,1,float16,float16,0,0.05146666864554087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,64,0,1,float16,float16,0,0.04580266773700714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,64,128,1,float16,fp8,0,0.050101334849993386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,64,128,1,fp8,fp8,0,0.06235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,64,0,1,float16,fp8,0,0.045994664231936135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,64,0,1,fp8,fp8,0,0.0414986660083135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,64,128,1,float16,float16,0,0.05023466547330221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,64,0,1,float16,float16,0,0.046223998069763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,64,128,1,float16,fp8,0,0.05085866649945577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,64,128,1,fp8,fp8,0,0.06397866706053416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,64,0,1,float16,fp8,0,0.04650133351484934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,64,0,1,fp8,fp8,0,0.04156800111134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,64,128,1,float16,float16,0,0.051039998730023704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,64,0,1,float16,float16,0,0.046298667788505554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,64,128,1,float16,fp8,0,0.051407997806866966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,64,128,1,fp8,fp8,0,0.06324266890684764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,64,0,1,float16,fp8,0,0.04677333434422811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,64,0,1,fp8,fp8,0,0.041589332123597465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,64,128,1,float16,float16,0,2.8401759465535483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,64,0,1,float16,float16,0,2.7744054794311523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,64,128,1,float16,fp8,0,2.85808531443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,64,0,1,fp8,fp8,0,2.550997257232666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,64,0,1,float16,fp8,0,2.7678133646647134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,64,128,1,float16,float16,0,2.900304158528646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,64,128,1,fp8,fp8,0,3.8794453938802085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,64,0,1,float16,float16,0,2.831439971923828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,64,128,1,float16,fp8,0,2.867018699645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,64,0,1,fp8,fp8,0,2.5393013954162598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,64,128,1,float16,float16,0,2.913344065348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,64,0,1,float16,fp8,0,2.7937065760294595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,64,128,1,fp8,fp8,0,3.891845385233561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,64,0,1,float16,float16,0,2.8369547526041665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,64,128,1,float16,fp8,0,2.920021375020345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,64,128,1,fp8,fp8,0,3.8997653325398765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,64,0,1,float16,fp8,0,2.8530400594075522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,64,0,1,fp8,fp8,0,2.570026715596517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,64,128,1,float16,float16,0,3.012960116068522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,64,0,1,float16,float16,0,2.9316959381103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,64,128,1,float16,fp8,0,3.0126771926879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,64,128,1,float16,float16,0,1.8639200528462727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,64,0,1,float16,fp8,0,2.9363040924072266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,64,0,1,fp8,fp8,0,2.617722670237223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,64,128,1,fp8,fp8,0,3.9596481323242188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,64,0,1,float16,float16,0,1.8420054117838542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,64,128,1,float16,fp8,0,1.7946186065673828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,64,128,1,fp8,fp8,0,2.2029760678609214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,64,0,1,float16,fp8,0,1.7857173283894856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,64,0,1,float16,float16,0,1.3578346570332844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,64,0,1,fp8,fp8,0,1.522821267445882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,64,128,1,float16,float16,0,1.403557300567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,64,128,1,float16,fp8,0,1.4418613115946453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,64,0,1,float16,fp8,0,1.3634293874104817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,64,128,1,fp8,fp8,0,1.9310720761617024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,64,0,1,fp8,fp8,0,1.2775253454844158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,64,128,1,float16,float16,0,1.4154292742411296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,64,0,1,float16,float16,0,1.3908692995707195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,64,128,1,float16,fp8,0,1.4397385915120442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,64,0,1,float16,fp8,0,1.3877280553181965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,64,128,1,fp8,fp8,0,1.9568212827046711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,64,0,1,fp8,fp8,0,1.2674986521402996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,64,128,1,float16,float16,0,1.4498079617818196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,64,0,1,float16,float16,0,1.3917867342631023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,64,128,1,float16,fp8,0,1.4390346209208171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,64,128,1,fp8,fp8,0,1.9584320386250813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,64,0,1,fp8,fp8,0,1.273919979731242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,64,0,1,float16,fp8,0,1.422320048014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,64,128,1,float16,float16,0,1.5045812924702961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,64,0,1,float16,float16,0,1.463578701019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,64,128,1,float16,fp8,0,1.504949410756429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,64,128,1,float16,float16,0,0.9347893397013346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,64,0,1,float16,fp8,0,1.4580319722493489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,64,0,1,fp8,fp8,0,1.3317493597666423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,64,128,1,fp8,fp8,0,1.9977280298868816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,64,0,1,float16,float16,0,0.9191786448160807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,64,128,1,float16,fp8,0,0.9023359616597494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,64,128,1,fp8,fp8,0,1.1089866956075032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,64,0,1,float16,fp8,0,0.8838933308919271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,64,0,1,float16,fp8,0,0.706005334854126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,64,0,1,fp8,fp8,0,0.7662399609883627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,64,128,1,float16,float16,0,0.7176799774169922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,64,0,1,float16,float16,0,0.7008693218231201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,64,0,1,float16,float16,0,0.7120800018310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,64,128,1,float16,fp8,0,0.721290667851766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,64,128,1,fp8,fp8,0,0.9858720302581787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,64,0,1,fp8,fp8,0,0.6442186832427979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,64,128,1,float16,float16,0,0.725488026936849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,64,128,1,float16,fp8,0,0.7280639807383219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,64,128,1,fp8,fp8,0,0.9874773025512695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,64,0,1,float16,fp8,0,0.7104480266571045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,64,0,1,fp8,fp8,0,0.6473600069681803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,64,128,1,float16,float16,0,0.7289973100026449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,64,0,1,float16,float16,0,0.7119680245717367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,64,128,1,float16,fp8,0,0.7306133111317953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,64,128,1,fp8,fp8,0,0.988912026087443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,64,0,1,float16,fp8,0,0.7120373249053955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,64,0,1,fp8,fp8,0,0.6472853422164917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,64,128,1,float16,float16,0,0.7414560317993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,64,0,1,float16,float16,0,0.7159252961476644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,64,128,1,float16,fp8,0,0.7368373076121012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,64,128,1,fp8,fp8,0,1.001253366470337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,64,0,1,float16,fp8,0,0.7211253643035889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,64,0,1,fp8,fp8,0,0.6621173222859701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,64,128,1,float16,float16,0,0.47988800207773846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,64,0,1,float16,float16,0,0.4710880120595296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,64,128,1,float16,fp8,0,0.4663519859313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,64,128,1,float16,fp8,0,0.37485333283742267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,64,128,1,fp8,fp8,0,0.558079997698466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,64,0,1,float16,fp8,0,0.45666666825612384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,64,0,1,fp8,fp8,0,0.33449065685272217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,64,0,1,fp8,fp8,0,0.3858400185902913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,64,128,1,float16,float16,0,0.3745386600494385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,64,0,1,float16,float16,0,0.3665119806925456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,64,128,1,fp8,fp8,0,0.5034559965133667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,64,0,1,float16,fp8,0,0.36509867509206134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,64,128,1,float16,float16,0,0.3778560161590576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,64,0,1,float16,float16,0,0.36739198366800946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,64,128,1,float16,fp8,0,0.3771200180053711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,64,128,1,fp8,fp8,0,0.5052853425343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,64,0,1,float16,fp8,0,0.36900798479715985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,64,0,1,fp8,fp8,0,0.34371201197306317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,64,128,1,float16,float16,0,0.3773546616236369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,64,0,1,float16,float16,0,0.3688533306121826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,64,128,1,float16,fp8,0,0.3805493513743083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,64,128,1,fp8,fp8,0,0.5061066548029581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,64,0,1,float16,fp8,0,0.37003199259440106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,64,0,1,fp8,fp8,0,0.33717866738637287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,64,128,1,float16,float16,0,0.2543626626332601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,64,128,1,float16,float16,0,0.381386677424113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,64,0,1,float16,float16,0,0.37562131881713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,64,128,1,float16,fp8,0,0.38074668248494464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,64,128,1,fp8,fp8,0,0.5081119934717814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,64,0,1,float16,fp8,0,0.37577064832051593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,64,0,1,fp8,fp8,0,0.3394613265991211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,64,0,1,float16,float16,0,0.25094934304555255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,64,128,1,float16,fp8,0,0.24769065777460733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,64,128,1,fp8,fp8,0,0.28878400723139447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,64,0,1,float16,fp8,0,0.24441067377726236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,64,0,1,fp8,fp8,0,0.20625066757202148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,64,128,1,float16,float16,0,0.19667200247446695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,64,0,1,fp8,fp8,0,0.18224000930786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,64,0,1,float16,float16,0,0.19348265727361044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,64,128,1,float16,fp8,0,0.19778132438659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,64,128,1,fp8,fp8,0,0.2633440097173055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,64,0,1,float16,fp8,0,0.19106666247049967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,64,128,1,float16,float16,0,0.19699732462565103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,64,0,1,float16,float16,0,0.19244267543156943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,64,128,1,float16,fp8,0,0.1991573373476664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,64,128,1,fp8,fp8,0,0.26530667146046955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,64,0,1,float16,fp8,0,0.19264533122380575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,64,128,1,fp8,fp8,0,0.2659466663996379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,64,0,1,fp8,fp8,0,0.18066666523615518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,64,0,1,fp8,fp8,0,0.18202133973439535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,64,128,1,float16,float16,0,0.19858133792877197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,64,0,1,float16,float16,0,0.1965013345082601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,64,128,1,float16,fp8,0,0.20012267430623373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,64,0,1,float16,float16,0,0.1933013399442037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,64,0,1,fp8,fp8,0,0.18368534247080484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,64,128,1,float16,fp8,0,0.1989333430926005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,64,0,1,float16,float16,0,0.13890133301417032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,64,128,1,float16,fp8,0,0.1360213359196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,64,0,1,float16,fp8,0,0.19489065806070963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,64,128,1,float16,float16,0,0.2015786568323771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,64,128,1,float16,float16,0,0.10610666871070862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,64,0,1,float16,fp8,0,0.19638399283091226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,64,128,1,fp8,fp8,0,0.26473599672317505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,64,128,1,float16,float16,0,0.1397546629110972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,64,128,1,fp8,fp8,0,0.15516799688339233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,64,0,1,float16,fp8,0,0.13577600320180258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,64,0,1,fp8,fp8,0,0.11601600050926208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,64,0,1,float16,float16,0,0.10280533631642659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,64,128,1,float16,fp8,0,0.10634133219718933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,64,128,1,fp8,fp8,0,0.14130133390426636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,64,0,1,float16,fp8,0,0.10357333223025005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,64,0,1,fp8,fp8,0,0.10086933771769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,64,128,1,float16,float16,0,0.10564266641934712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,64,0,1,float16,float16,0,0.10382933417956035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,64,128,1,float16,fp8,0,0.10571199655532837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,64,128,1,fp8,fp8,0,0.14222932855288187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,64,0,1,float16,fp8,0,0.10348266363143921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,64,0,1,fp8,fp8,0,0.10016533732414246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,64,128,1,float16,float16,0,0.10614933570226033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,64,0,1,float16,float16,0,0.1029813289642334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,64,128,1,float16,fp8,0,0.10655466715494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,64,128,1,fp8,fp8,0,0.1418293317159017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,64,0,1,float16,fp8,0,0.1034453312555949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,64,0,1,fp8,fp8,0,0.10204799969991048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,64,128,1,float16,float16,0,0.10790933171908061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,64,0,1,float16,float16,0,0.10473600029945374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,64,128,1,float16,fp8,0,0.10699733098347981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,64,128,1,fp8,fp8,0,0.14299733440081278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,64,0,1,float16,fp8,0,0.10428266723950703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,64,0,1,fp8,fp8,0,0.10341333349545796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,64,128,1,float16,float16,0,0.08172266681989034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,64,0,1,float16,float16,0,0.08116266628106435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,64,128,1,float16,fp8,0,0.08078933258851369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,64,128,1,fp8,fp8,0,0.08809066812197368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,64,0,1,float16,fp8,0,0.07973333199818929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,64,0,1,fp8,fp8,0,0.06824000179767609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,64,128,1,float16,float16,0,0.0609386662642161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,64,0,1,float16,float16,0,0.062447999914487205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,64,128,1,float16,fp8,0,0.061861331264177956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,64,128,1,fp8,fp8,0,0.07736533383528392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,64,0,1,float16,fp8,0,0.06229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,64,0,1,fp8,fp8,0,0.05638933181762695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,64,128,1,float16,float16,0,0.061887999375661217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,64,0,1,float16,float16,0,0.06246933341026306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,64,128,1,float16,fp8,0,0.06196799874305725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,64,128,1,fp8,fp8,0,0.07745066781838734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,64,0,1,float16,fp8,0,0.062128002444903054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,64,0,1,fp8,fp8,0,0.05707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,64,128,1,float16,float16,0,0.06252799928188324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,64,0,1,float16,float16,0,0.061290666460990906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,64,128,1,float16,fp8,0,0.06181866427262624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,64,128,1,fp8,fp8,0,0.07784000039100647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,64,0,1,float16,fp8,0,0.06201066573460897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,64,0,1,fp8,fp8,0,0.05675200124581655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,64,128,1,float16,float16,0,0.06158933540185293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,64,0,1,float16,float16,0,0.06353599826494853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,64,0,1,float16,float16,0,0.04364266494909922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,64,128,1,float16,fp8,0,0.06180266539255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,64,128,1,fp8,fp8,0,0.07928533355395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,64,0,1,float16,fp8,0,0.061850666999816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,64,0,1,fp8,fp8,0,0.058245331048965454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,64,128,1,float16,float16,0,0.0436160018046697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,64,128,1,float16,fp8,0,0.04334933559099833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,64,128,1,fp8,fp8,0,0.05253333350022634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,64,0,1,float16,fp8,0,0.043247997760772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,64,0,1,fp8,fp8,0,0.041850666205088295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,64,128,1,float16,float16,0,0.03987200061480204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,64,0,1,float16,float16,0,0.038848000268141426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,64,128,1,float16,fp8,0,0.040021332601706185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,64,128,1,fp8,fp8,0,0.04734399914741516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,64,0,1,float16,fp8,0,0.03892799963553747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,64,0,1,fp8,fp8,0,0.036831999818483986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,64,0,1,fp8,fp8,0,0.03729599962631861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,64,128,1,float16,float16,0,0.0403466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,64,0,1,float16,float16,0,0.03925866633653641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,64,128,1,float16,fp8,0,0.040005333721637726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,64,128,1,float16,fp8,0,0.04085333396991094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,64,128,1,fp8,fp8,0,0.04659733176231384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,64,0,1,float16,fp8,0,0.040074666341145836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,64,128,1,float16,float16,0,0.04037333279848099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,64,0,1,float16,float16,0,0.03944533318281174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,64,128,1,fp8,fp8,0,0.04762133459250132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,64,0,1,float16,fp8,0,0.03923733284076055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,64,0,1,fp8,fp8,0,0.037445334096749626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,64,128,1,float16,float16,0,0.0402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,64,0,1,float16,float16,0,0.03915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,64,128,1,float16,fp8,0,0.04035733391841253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,64,0,1,float16,float16,0,0.030506665507952373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,64,128,1,float16,fp8,0,0.0310506671667099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,64,128,1,fp8,fp8,0,0.04739200075467428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,64,0,1,float16,fp8,0,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,64,0,1,fp8,fp8,0,0.036992001036802925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,64,0,1,float16,float16,0,0.02903999884923299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,64,128,1,float16,float16,0,0.03156800071398417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,64,128,1,fp8,fp8,0,0.0355679988861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,64,0,1,fp8,fp8,0,0.029359998802344005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,64,0,1,float16,fp8,0,0.028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,64,0,1,float16,fp8,0,0.03065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,64,128,1,float16,float16,0,0.028778667251269024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,64,128,1,float16,float16,0,0.02871999889612198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,64,128,1,float16,fp8,0,0.028783999383449554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,64,128,1,fp8,fp8,0,0.03495466709136963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,64,0,1,fp8,fp8,0,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,64,0,1,float16,float16,0,0.028117333849271137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,64,128,1,float16,fp8,0,0.02903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,64,128,1,fp8,fp8,0,0.034458667039871216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,64,0,1,float16,fp8,0,0.02790933350721995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,64,0,1,fp8,fp8,0,0.027034667630990345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,64,128,1,float16,float16,0,0.029130667448043823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,64,0,1,float16,float16,0,0.028031999866167705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,64,128,1,float16,fp8,0,0.02916266769170761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,64,0,1,float16,fp8,0,0.028677334388097126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,64,128,1,fp8,fp8,0,0.03460799902677536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,64,0,1,fp8,fp8,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,64,128,1,float16,float16,0,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,64,0,1,float16,float16,0,0.028170667588710785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,64,128,1,float16,fp8,0,0.029066666960716248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,64,128,1,fp8,fp8,0,0.03448000053564707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,64,0,1,float16,fp8,0,0.028431999186674755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,64,0,1,fp8,fp8,0,0.027935999135176342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,64,128,1,float16,float16,0,1.0461280345916748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,64,0,1,float16,float16,0,1.0384480158487956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,64,128,1,float16,fp8,0,1.0399893124898274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,64,128,1,fp8,fp8,0,1.2552426656087239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,64,0,1,float16,fp8,0,1.0460000038146973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,64,128,1,float16,float16,0,1.0649706522623699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,64,0,1,fp8,fp8,0,1.256661335627238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,64,0,1,float16,float16,0,1.060421307881673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,64,128,1,float16,fp8,0,1.0577119986216228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,64,0,1,float16,fp8,0,1.054645299911499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,64,128,1,fp8,fp8,0,1.2611520290374756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,64,128,1,float16,float16,0,1.0713600317637126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,64,0,1,fp8,fp8,0,1.2650346755981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,64,0,1,float16,float16,0,1.0707893371582031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,64,128,1,float16,fp8,0,1.0517226854960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,64,0,1,float16,fp8,0,1.050437370936076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,64,128,1,fp8,fp8,0,1.2644106547037761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,64,128,1,float16,float16,0,1.1231253147125244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,64,0,1,fp8,fp8,0,1.2562560240427654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,64,0,1,float16,float16,0,1.1259893576304119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,64,128,1,float16,fp8,0,1.1201866467793782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,64,128,1,float16,float16,0,0.7743519941965739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,64,0,1,float16,fp8,0,1.1193919976552327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,64,128,1,fp8,fp8,0,1.3111680348714192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,64,0,1,float16,fp8,0,0.7399253050486246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,64,0,1,fp8,fp8,0,1.3149813016255696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,64,0,1,float16,float16,0,0.7717493375142416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,64,128,1,float16,fp8,0,0.7408959865570068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,64,128,1,fp8,fp8,0,0.7663946946461996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,64,0,1,fp8,fp8,0,0.773296038309733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,64,128,1,float16,float16,0,0.5314506689707438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,64,0,1,float16,fp8,0,0.5225973526636759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,64,0,1,float16,float16,0,0.5224266846974691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,64,128,1,float16,fp8,0,0.5252213478088379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,64,128,1,fp8,fp8,0,0.6396373510360718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,64,0,1,fp8,fp8,0,0.6293866634368896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,64,128,1,float16,float16,0,0.530778686205546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,64,0,1,float16,float16,0,0.5391466617584229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,64,128,1,float16,fp8,0,0.5388000011444092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,64,0,1,float16,float16,0,0.5378506580988566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,64,128,1,fp8,fp8,0,0.6364533503850301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,64,0,1,float16,fp8,0,0.5279733339945475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,64,0,1,fp8,fp8,0,0.6360640128453573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,64,128,1,float16,float16,0,0.5348426500956217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,64,128,1,float16,fp8,0,0.5381333430608114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,64,128,1,fp8,fp8,0,0.6483360131581625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,64,0,1,float16,fp8,0,0.5346399943033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,64,0,1,fp8,fp8,0,0.6412853399912516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,64,128,1,float16,float16,0,0.5426453351974487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,64,0,1,float16,float16,0,0.5457866589228312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,64,128,1,float16,fp8,0,0.5403466622034708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,64,128,1,fp8,fp8,0,0.6664906740188599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,64,0,1,float16,fp8,0,0.548906683921814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,64,0,1,fp8,fp8,0,0.6648639837900797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,64,128,1,float16,float16,0,0.39450665314992267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,64,0,1,float16,float16,0,0.3940800031026204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,64,128,1,float16,fp8,0,0.38154133160909015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,64,128,1,fp8,fp8,0,0.38745065530141193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,64,0,1,float16,fp8,0,0.37916799386342365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,64,0,1,fp8,fp8,0,0.386954665184021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,64,128,1,float16,float16,0,0.27459200223286945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,64,0,1,float16,fp8,0,0.272053341070811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,64,0,1,float16,float16,0,0.27317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,64,128,1,float16,float16,0,0.2735946575800578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,64,128,1,float16,fp8,0,0.27266667286554974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,64,128,1,fp8,fp8,0,0.3304799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,64,0,1,fp8,fp8,0,0.3295573393503825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,64,0,1,float16,float16,0,0.2743946711222331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,64,128,1,float16,fp8,0,0.27380800247192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,64,128,1,fp8,fp8,0,0.33187200625737506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,64,0,1,float16,fp8,0,0.2740853428840637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,64,0,1,fp8,fp8,0,0.3324586749076843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,64,128,1,float16,float16,0,0.2747466762860616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,64,0,1,float16,float16,0,0.2753973404566447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,64,128,1,float16,fp8,0,0.2749013304710388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,64,128,1,fp8,fp8,0,0.3313279946645101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,64,0,1,float16,fp8,0,0.27508799235026044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,64,0,1,fp8,fp8,0,0.33050666252772015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,64,128,1,float16,float16,0,0.2821386655171712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,64,0,1,float16,float16,0,0.28195200363794964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,64,128,1,float16,fp8,0,0.2805440028508504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,64,128,1,fp8,fp8,0,0.33934934933980304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,64,0,1,float16,fp8,0,0.28173865874608356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,64,0,1,fp8,fp8,0,0.3394720156987508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,64,128,1,float16,float16,0,0.20910932620366415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,64,128,1,float16,float16,0,0.14412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,64,0,1,float16,float16,0,0.2078933318456014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,64,128,1,float16,fp8,0,0.20060799519220987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,64,128,1,fp8,fp8,0,0.205295999844869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,64,0,1,float16,fp8,0,0.20042133331298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,64,0,1,fp8,fp8,0,0.20485333601633707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,64,0,1,float16,float16,0,0.14432000120480856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,64,128,1,float16,fp8,0,0.1442506710688273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,64,128,1,fp8,fp8,0,0.17725332578023276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,64,0,1,float16,fp8,0,0.1437493364016215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,64,0,1,fp8,fp8,0,0.17811199029286703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,64,128,1,float16,float16,0,0.14493333299954733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,64,0,1,float16,float16,0,0.14471466342608133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,64,128,1,float16,fp8,0,0.14542933305104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,64,128,1,fp8,fp8,0,0.17781333128611246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,64,0,1,float16,fp8,0,0.14524799585342407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,64,0,1,fp8,fp8,0,0.17796266078948975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,64,128,1,float16,float16,0,0.145797332127889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,64,0,1,float16,float16,0,0.14637866616249084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,64,128,1,float16,fp8,0,0.14550933241844177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,64,128,1,float16,fp8,0,0.1480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,64,128,1,fp8,fp8,0,0.1798293391863505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,64,128,1,fp8,fp8,0,0.1786293387413025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,64,0,1,fp8,fp8,0,0.17999466260274252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,64,0,1,float16,fp8,0,0.14589333534240723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,64,0,1,fp8,fp8,0,0.1775839924812317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,64,0,1,float16,float16,0,0.14890133341153464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,64,128,1,fp8,fp8,0,0.11366933584213257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,64,128,1,float16,float16,0,0.14772799611091614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,64,0,1,float16,fp8,0,0.14787200093269348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,64,128,1,float16,float16,0,0.11371200283368428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,64,0,1,float16,float16,0,0.11362666885058086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,64,128,1,float16,fp8,0,0.11054933071136475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,64,0,1,float16,fp8,0,0.10976533095041911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,64,0,1,fp8,fp8,0,0.1145919958750407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,64,128,1,float16,float16,0,0.07996266583601634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,64,0,1,float16,float16,0,0.07971199850241344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,64,128,1,float16,fp8,0,0.08017600079377492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,64,128,1,fp8,fp8,0,0.09973866740862529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,64,0,1,float16,fp8,0,0.0795253316561381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,64,0,1,fp8,fp8,0,0.10044800241788228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,64,128,1,float16,float16,0,0.08011733492215474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,64,0,1,float16,float16,0,0.08085333307584126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,64,128,1,float16,fp8,0,0.08041599889596303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,64,128,1,fp8,fp8,0,0.10028800368309021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,64,0,1,float16,fp8,0,0.08036800225575765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,64,0,1,fp8,fp8,0,0.10019200046857198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,64,128,1,float16,float16,0,0.08077866832415263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,64,0,1,float16,float16,0,0.08029866715272267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,64,128,1,float16,fp8,0,0.08111466467380524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,64,128,1,fp8,fp8,0,0.10060800115267436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,64,0,1,float16,fp8,0,0.08142933249473572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,64,0,1,fp8,fp8,0,0.1006666620572408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,64,128,1,float16,float16,0,0.08158933122952779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,64,0,1,float16,float16,0,0.08122133215268452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,64,128,1,float16,fp8,0,0.0817386656999588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,64,128,1,fp8,fp8,0,0.1014400025208791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,64,0,1,float16,fp8,0,0.08205333352088928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,64,0,1,fp8,fp8,0,0.10154133041699727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,64,128,1,float16,float16,0,0.06177066763242086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,64,0,1,float16,float16,0,0.062447999914487205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,64,128,1,float16,fp8,0,0.0605973352988561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,64,128,1,fp8,fp8,0,0.06717866659164429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,64,0,1,float16,fp8,0,0.060175999999046326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,64,0,1,fp8,fp8,0,0.06667733192443848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,64,128,1,float16,float16,0,0.046298667788505554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,64,0,1,float16,float16,0,0.04610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,64,0,1,float16,float16,0,0.04589866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,64,128,1,float16,fp8,0,0.04598399996757507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,64,128,1,fp8,fp8,0,0.056373332937558494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,64,0,1,float16,fp8,0,0.046613335609436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,64,0,1,fp8,fp8,0,0.05696000158786774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,64,128,1,float16,float16,0,0.046816001335779824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,64,128,1,float16,fp8,0,0.04598933458328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,64,128,1,fp8,fp8,0,0.05801066756248474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,64,0,1,float16,fp8,0,0.04635733366012573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,64,128,1,fp8,fp8,0,0.05704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,64,0,1,fp8,fp8,0,0.05657066901524862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,64,128,1,float16,float16,0,0.04712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,64,0,1,float16,float16,0,0.04630400240421295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,64,128,1,float16,fp8,0,0.04703466594219208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,64,0,1,float16,fp8,0,0.04731733103593191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,64,0,1,fp8,fp8,0,0.057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,64,128,1,float16,float16,0,0.0470773329337438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,64,0,1,float16,float16,0,0.04708800216515859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,64,128,1,float16,fp8,0,0.04735999802748362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,64,128,1,fp8,fp8,0,0.05820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,64,0,1,float16,fp8,0,0.04782933493455251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,64,0,1,fp8,fp8,0,0.05834666887919108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,64,128,1,float16,float16,0,0.03323200096686681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,64,0,1,float16,float16,0,0.03293866664171219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,64,128,1,float16,fp8,0,0.03348266581694285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,64,128,1,fp8,fp8,0,0.041477332512537636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,64,0,1,float16,fp8,0,0.03257066756486893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,64,0,1,fp8,fp8,0,0.04160533348719279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,64,128,1,float16,float16,0,0.029535998900731403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,64,0,1,float16,float16,0,0.029616000751654308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,64,128,1,float16,fp8,0,0.02998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,64,128,1,fp8,fp8,0,0.03608000030120214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,64,0,1,float16,fp8,0,0.03035199890534083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,64,0,1,fp8,fp8,0,0.03636800001064936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,64,128,1,float16,float16,0,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,64,0,1,float16,float16,0,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,64,128,1,float16,fp8,0,0.02985599885384242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,64,128,1,fp8,fp8,0,0.03656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,64,0,1,float16,fp8,0,0.029461334149042766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,64,0,1,fp8,fp8,0,0.03626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,64,128,1,float16,float16,0,0.03011200080315272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,64,0,1,float16,float16,0,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,64,128,1,float16,fp8,0,0.030159999926884968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,64,128,1,float16,fp8,0,0.03012799968322118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,64,128,1,fp8,fp8,0,0.03633599976698557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,64,0,1,float16,fp8,0,0.030159999926884968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,64,0,1,fp8,fp8,0,0.03695466617743174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,64,128,1,float16,float16,0,0.02934933453798294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,64,0,1,float16,float16,0,0.029824001093705494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,64,128,1,float16,fp8,0,0.02329600105683009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,64,128,1,fp8,fp8,0,0.02848000079393387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,64,128,1,fp8,fp8,0,0.036848001182079315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,64,0,1,float16,fp8,0,0.030016000072161358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,64,0,1,fp8,fp8,0,0.036874666810035706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,64,128,1,float16,float16,0,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,64,0,1,float16,float16,0,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,64,0,1,float16,fp8,0,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,64,0,1,fp8,fp8,0,0.02850666642189026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,64,128,1,float16,float16,0,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,64,0,1,float16,float16,0,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,64,128,1,float16,fp8,0,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,64,128,1,fp8,fp8,0,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,64,0,1,float16,fp8,0,0.022101332743962605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,64,0,1,fp8,fp8,0,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,64,128,1,float16,float16,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,64,0,1,float16,float16,0,0.022154666483402252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,64,128,1,float16,fp8,0,0.021744000415007275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,64,128,1,fp8,fp8,0,0.026949333647886913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,64,0,1,float16,fp8,0,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,64,0,1,fp8,fp8,0,0.026954665780067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,64,128,1,float16,float16,0,0.022101332743962605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,64,0,1,float16,float16,0,0.02199999988079071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,64,128,1,float16,fp8,0,0.022143999735514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,64,128,1,fp8,fp8,0,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,64,0,1,float16,fp8,0,0.02182399978240331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,64,0,1,fp8,fp8,0,0.027034667630990345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,64,128,1,float16,float16,0,0.021935999393463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,64,0,1,float16,float16,0,0.021856000026067097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,64,128,1,float16,fp8,0,0.022629333039124806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,64,128,1,fp8,fp8,0,0.028090665737787884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,64,0,1,float16,fp8,0,0.022175999979178112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,64,0,1,fp8,fp8,0,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,64,128,1,float16,float16,0,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,64,0,1,float16,float16,0,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,64,128,1,float16,fp8,0,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,64,128,1,fp8,fp8,0,0.02040533348917961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,64,0,1,float16,fp8,0,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,64,0,1,fp8,fp8,0,0.02075733368595441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,64,128,1,float16,float16,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,64,0,1,float16,float16,0,0.015813333292802174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,64,128,1,float16,fp8,0,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,64,128,1,fp8,fp8,0,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,64,0,1,float16,fp8,0,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,64,0,1,fp8,fp8,0,0.020058666666348774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,64,128,1,float16,float16,0,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,64,0,1,float16,float16,0,0.01600533351302147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,64,0,1,float16,float16,0,0.01607999950647354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,64,128,1,float16,fp8,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,64,128,1,fp8,fp8,0,0.01977066695690155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,64,0,1,float16,fp8,0,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,64,128,1,fp8,fp8,0,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,64,0,1,float16,fp8,0,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,64,0,1,fp8,fp8,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,64,128,1,float16,fp8,0,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,64,128,1,float16,float16,0,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,64,128,1,float16,fp8,0,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,64,0,1,fp8,fp8,0,0.01970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,64,128,1,float16,float16,0,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,64,0,1,float16,float16,0,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,64,128,1,fp8,fp8,0,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,64,0,1,float16,fp8,0,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,64,0,1,fp8,fp8,0,0.019920000185569126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,64,128,1,float16,float16,0,0.4941706657409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,64,0,1,float16,float16,0,0.49564798672993976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,64,128,1,float16,fp8,0,0.49330135186513263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,64,128,1,fp8,fp8,0,0.8668159643809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,64,0,1,float16,fp8,0,0.493557333946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,64,0,1,fp8,fp8,0,0.8669386704762777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,64,128,1,float16,float16,0,0.49768535296122235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,64,0,1,float16,fp8,0,0.4982186555862427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,64,0,1,float16,float16,0,0.4996906518936157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,64,128,1,float16,fp8,0,0.5006346702575684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,64,0,1,fp8,fp8,0,0.8769546349843343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,64,128,1,fp8,fp8,0,0.8733386993408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,64,128,1,float16,float16,0,0.5019413232803345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,64,128,1,float16,fp8,0,0.4962186813354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,64,0,1,float16,float16,0,0.5002613464991251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,64,128,1,fp8,fp8,0,0.8768266836802164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,64,0,1,float16,fp8,0,0.5008000135421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,64,128,1,float16,float16,0,0.5261333386103312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,64,0,1,fp8,fp8,0,0.8716853459676107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,64,0,1,float16,float16,0,0.5291573206583658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,64,128,1,float16,fp8,0,0.5109013319015503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,64,0,1,float16,fp8,0,0.511189341545105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,64,128,1,fp8,fp8,0,0.8992533683776855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,64,128,1,float16,float16,0,0.3920746644337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,64,0,1,float16,float16,0,0.39098668098449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,64,128,1,float16,fp8,0,0.37614933649698895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,64,0,1,fp8,fp8,0,0.8978239695231119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,64,0,1,float16,fp8,0,0.3749653498331706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,64,128,1,fp8,fp8,0,0.5047253370285034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,64,0,1,fp8,fp8,0,0.5059946775436401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,64,128,1,float16,float16,0,0.2553226749102275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,64,0,1,float16,float16,0,0.2564479907353719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,64,128,1,float16,fp8,0,0.2563520073890686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,64,128,1,fp8,fp8,0,0.441861351331075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,64,0,1,float16,fp8,0,0.25516800085703534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,64,0,1,fp8,fp8,0,0.442959984143575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,64,128,1,float16,float16,0,0.25777600208918255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,64,0,1,float16,float16,0,0.2575146754582723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,64,128,1,float16,fp8,0,0.2574506600697835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,64,128,1,fp8,fp8,0,0.44841599464416504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,64,0,1,float16,fp8,0,0.2579360008239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,64,128,1,float16,float16,0,0.2571413318316142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,64,0,1,fp8,fp8,0,0.44401601950327557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,64,0,1,float16,float16,0,0.25836267073949176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,64,128,1,float16,fp8,0,0.2581226627031962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,64,128,1,fp8,fp8,0,0.44648532072703045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,64,0,1,float16,fp8,0,0.25841599702835083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,64,0,1,fp8,fp8,0,0.44889601071675617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,64,128,1,float16,float16,0,0.264138658841451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,64,0,1,float16,fp8,0,0.2634933392206828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,64,0,1,float16,float16,0,0.2633013327916463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,64,128,1,float16,fp8,0,0.26239466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,64,128,1,fp8,fp8,0,0.454095999399821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,64,128,1,float16,float16,0,0.2049973408381144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,64,128,1,float16,float16,0,0.13682132959365845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,64,0,1,fp8,fp8,0,0.4561813275019328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,64,0,1,float16,float16,0,0.20364266633987427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,64,128,1,float16,fp8,0,0.19638933738072714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,64,128,1,fp8,fp8,0,0.2603999972343445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,64,0,1,float16,fp8,0,0.19671465953191122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,64,0,1,fp8,fp8,0,0.26031466325124103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,64,128,1,float16,float16,0,0.1371999979019165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,64,0,1,float16,float16,0,0.13687466581662497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,64,128,1,float16,fp8,0,0.13716266552607217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,64,128,1,fp8,fp8,0,0.23406400283177695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,64,0,1,float16,fp8,0,0.13660800457000732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,64,0,1,fp8,fp8,0,0.23448532819747925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,64,0,1,float16,float16,0,0.13699199755986533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,64,128,1,float16,fp8,0,0.1362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,64,128,1,fp8,fp8,0,0.23463465770085654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,64,0,1,float16,fp8,0,0.13723733027776083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,64,0,1,fp8,fp8,0,0.23465599616368613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,64,128,1,float16,float16,0,0.13633066415786743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,64,0,1,float16,float16,0,0.13717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,64,128,1,float16,fp8,0,0.13753599921862283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,64,128,1,fp8,fp8,0,0.23449599742889404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,64,0,1,float16,fp8,0,0.1379680037498474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,64,0,1,fp8,fp8,0,0.23425066471099854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,64,128,1,float16,float16,0,0.14074132839838663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,64,0,1,fp8,fp8,0,0.23813867568969727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,64,0,1,float16,float16,0,0.14123732844988504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,64,128,1,float16,fp8,0,0.13954666256904602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,64,128,1,fp8,fp8,0,0.14125333229700723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,64,128,1,fp8,fp8,0,0.23751999934514365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,64,0,1,float16,fp8,0,0.1402773360411326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,64,128,1,float16,float16,0,0.110042671362559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,64,0,1,float16,float16,0,0.10940266648928325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,64,128,1,float16,fp8,0,0.10613866647084554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,64,0,1,float16,fp8,0,0.1055519978205363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,64,0,1,fp8,fp8,0,0.14089600245157877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,64,128,1,float16,float16,0,0.07409599920113881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,64,0,1,float16,float16,0,0.07377600173155467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,64,128,1,float16,fp8,0,0.07473066449165344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,64,128,1,fp8,fp8,0,0.12787200013796488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,64,128,1,float16,fp8,0,0.07449066638946533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,64,0,1,float16,fp8,0,0.0740533322095871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,64,0,1,fp8,fp8,0,0.12827199697494507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,64,128,1,float16,float16,0,0.07479466497898102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,64,0,1,float16,float16,0,0.07452799876530965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,64,128,1,fp8,fp8,0,0.12659733494122824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,64,0,1,float16,fp8,0,0.07482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,64,0,1,fp8,fp8,0,0.1279306709766388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,64,0,1,float16,fp8,0,0.07569600145022075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,64,128,1,float16,float16,0,0.07516799867153168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,64,0,1,float16,float16,0,0.07542933523654938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,64,0,1,float16,float16,0,0.07669333120187123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,64,128,1,float16,fp8,0,0.07680533329645793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,64,128,1,float16,fp8,0,0.07495999832948048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,64,128,1,fp8,fp8,0,0.12666133046150208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,64,0,1,fp8,fp8,0,0.12657599647839865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,64,128,1,float16,float16,0,0.07683200140794118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,64,128,1,fp8,fp8,0,0.129530668258667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,64,0,1,float16,fp8,0,0.07668800155321757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,64,0,1,fp8,fp8,0,0.1297866702079773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,64,128,1,float16,float16,0,0.05804799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,64,0,1,float16,float16,0,0.0577706644932429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,64,128,1,float16,fp8,0,0.05607999861240387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,64,0,1,float16,fp8,0,0.056032001972198486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,64,128,1,fp8,fp8,0,0.08084266881148021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,64,0,1,fp8,fp8,0,0.08019733428955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,64,128,1,float16,float16,0,0.04284800092379252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,64,0,1,float16,float16,0,0.042693331837654114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,64,128,1,float16,fp8,0,0.042805333932240806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,64,128,1,fp8,fp8,0,0.07189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,64,0,1,float16,fp8,0,0.042453333735466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,64,0,1,fp8,fp8,0,0.07196799914042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,64,128,1,float16,float16,0,0.042810668547948204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,64,128,1,float16,float16,0,0.04307733476161957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,64,0,1,float16,float16,0,0.042506664991378784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,64,128,1,float16,fp8,0,0.04265599946180979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,64,128,1,fp8,fp8,0,0.07157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,64,0,1,float16,fp8,0,0.04317333300908407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,64,0,1,float16,fp8,0,0.04271466533342997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,64,0,1,fp8,fp8,0,0.07198399802049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,64,0,1,fp8,fp8,0,0.07236800094445546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,64,0,1,float16,float16,0,0.04279999931653341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,64,128,1,float16,fp8,0,0.043696001172065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,64,128,1,fp8,fp8,0,0.07092266778151195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,64,128,1,float16,float16,0,0.0439626673857371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,64,0,1,float16,float16,0,0.04397333165009817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,64,128,1,float16,fp8,0,0.04348266621430715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,64,128,1,fp8,fp8,0,0.07076266904671986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,64,0,1,float16,fp8,0,0.043621331453323364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,64,0,1,fp8,fp8,0,0.07241599758466084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,64,128,1,float16,float16,0,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,64,0,1,float16,float16,0,0.03196800003449122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,64,128,1,float16,fp8,0,0.03107200066248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,64,128,1,fp8,fp8,0,0.04900800188382467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,64,0,1,float16,fp8,0,0.030506665507952373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,64,0,1,fp8,fp8,0,0.04887466629346212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,64,128,1,float16,float16,0,0.028207999964555103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,64,0,1,float16,float16,0,0.027717334528764088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,64,128,1,float16,fp8,0,0.02845333268245061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,64,128,1,fp8,fp8,0,0.04419733087221781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,64,0,1,float16,fp8,0,0.028064000109831493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,64,0,1,fp8,fp8,0,0.044677332043647766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,64,128,1,float16,float16,0,0.02812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,64,0,1,float16,float16,0,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,64,128,1,float16,fp8,0,0.02789866675933202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,64,128,1,fp8,fp8,0,0.04483733574549357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,64,0,1,float16,fp8,0,0.028192001084486645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,64,0,1,fp8,fp8,0,0.044138665000597634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,64,128,1,float16,float16,0,0.02809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,64,0,1,float16,float16,0,0.028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,64,128,1,float16,fp8,0,0.028357334434986115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,64,128,1,fp8,fp8,0,0.04442666471004486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,64,0,1,float16,fp8,0,0.028565332293510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,64,0,1,fp8,fp8,0,0.04443199932575226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,64,128,1,float16,float16,0,0.02844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,64,0,1,float16,float16,0,0.027957332630952198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,64,128,1,float16,fp8,0,0.029274667302767437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,64,128,1,fp8,fp8,0,0.044639999667803444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,64,0,1,float16,fp8,0,0.028378665447235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,64,0,1,fp8,fp8,0,0.04433600107828776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,64,128,1,float16,float16,0,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,64,0,1,float16,float16,0,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,64,128,1,float16,fp8,0,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,64,128,1,fp8,fp8,0,0.0305173322558403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,64,0,1,float16,fp8,0,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,64,0,1,fp8,fp8,0,0.031008000175158184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,64,128,1,float16,float16,0,0.019685332973798115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,64,0,1,float16,float16,0,0.019733333339293797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,64,128,1,float16,fp8,0,0.02022933339079221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,64,128,1,fp8,fp8,0,0.02938666691382726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,64,0,1,float16,fp8,0,0.01993600030740102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,64,0,1,fp8,fp8,0,0.0317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,64,128,1,float16,float16,0,0.019909333437681198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,64,0,1,float16,float16,0,0.019674666225910187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,64,128,1,float16,fp8,0,0.02019199977318446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,64,128,1,fp8,fp8,0,0.029898665845394135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,64,0,1,float16,fp8,0,0.02015999952952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,64,0,1,fp8,fp8,0,0.029839999973773956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,64,128,1,float16,float16,0,0.019861333072185516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,64,0,1,float16,float16,0,0.019989332805077236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,64,128,1,float16,fp8,0,0.019754666835069656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,64,128,1,fp8,fp8,0,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,64,0,1,float16,fp8,0,0.02025066688656807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,64,0,1,fp8,fp8,0,0.029898665845394135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,64,128,1,float16,float16,0,0.02004266654451688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,64,0,1,float16,float16,0,0.020010666300853092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,64,128,1,float16,fp8,0,0.02041600023706754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,64,128,1,fp8,fp8,0,0.029701332251230877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,64,0,1,float16,fp8,0,0.020015999674797058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,64,0,1,fp8,fp8,0,0.03002133220434189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,64,128,1,float16,float16,0,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,64,0,1,float16,float16,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,64,128,1,float16,fp8,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,64,128,1,fp8,fp8,0,0.023408000667889912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,64,0,1,float16,fp8,0,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,64,0,1,fp8,fp8,0,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,64,128,1,float16,float16,0,0.014490666488806406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,64,0,1,float16,float16,0,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,64,128,1,float16,fp8,0,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,64,128,1,fp8,fp8,0,0.02253866692384084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,64,0,1,float16,fp8,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,64,0,1,fp8,fp8,0,0.022570667167504627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,64,128,1,float16,float16,0,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,64,0,1,float16,float16,0,0.014469332993030548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,64,128,1,float16,fp8,0,0.014650666465361914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,64,128,1,fp8,fp8,0,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,64,0,1,float16,fp8,0,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,64,0,1,fp8,fp8,0,0.022570667167504627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,64,128,1,float16,float16,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,64,0,1,float16,float16,0,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,64,128,1,float16,fp8,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,64,128,1,fp8,fp8,0,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,64,0,1,float16,fp8,0,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,64,0,1,fp8,fp8,0,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,64,128,1,float16,float16,0,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,64,0,1,float16,float16,0,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,64,128,1,float16,fp8,0,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,64,128,1,fp8,fp8,0,0.022533332308133442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,64,0,1,float16,fp8,0,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,64,0,1,fp8,fp8,0,0.022709332406520844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,64,128,1,float16,float16,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,64,0,1,float16,float16,0,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,64,128,1,float16,fp8,0,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,64,128,1,fp8,fp8,0,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,64,0,1,float16,fp8,0,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,64,0,1,fp8,fp8,0,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,64,128,1,float16,float16,0,0.012608000387748083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,64,0,1,float16,float16,0,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,64,128,1,float16,fp8,0,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,64,128,1,fp8,fp8,0,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,64,0,1,float16,fp8,0,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,64,0,1,fp8,fp8,0,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,64,128,1,float16,float16,0,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,64,0,1,float16,float16,0,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,64,128,1,float16,fp8,0,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,64,128,1,fp8,fp8,0,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,64,0,1,float16,fp8,0,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,64,0,1,fp8,fp8,0,0.01851733277241389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,64,128,1,float16,float16,0,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,64,0,1,float16,float16,0,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,64,128,1,float16,fp8,0,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,64,128,1,fp8,fp8,0,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,64,0,1,float16,fp8,0,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,64,0,1,fp8,fp8,0,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,64,128,1,float16,float16,0,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,64,0,1,float16,float16,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,64,128,1,float16,fp8,0,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,64,128,1,fp8,fp8,0,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,64,0,1,float16,fp8,0,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,64,0,1,fp8,fp8,0,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,64,128,1,float16,float16,0,0.288698673248291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,64,0,1,float16,float16,0,0.2890773415565491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,64,128,1,float16,fp8,0,0.286901334921519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,64,0,1,float16,fp8,0,0.28758400678634644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,64,128,1,fp8,fp8,0,0.721834659576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,64,128,1,float16,float16,0,0.291103998819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,64,0,1,fp8,fp8,0,0.7200533548990885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,64,0,1,float16,float16,0,0.2900000015894572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,64,128,1,float16,fp8,0,0.2914026578267415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,64,0,1,float16,fp8,0,0.2903413375218709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,64,128,1,fp8,fp8,0,0.7294346491495768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,64,128,1,float16,float16,0,0.2912213404973348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,64,0,1,float16,float16,0,0.2905120054880778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,64,0,1,fp8,fp8,0,0.7223359743754069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,64,0,1,fp8,fp8,0,0.7233973344167074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,64,128,1,float16,fp8,0,0.2908160090446472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,64,0,1,float16,fp8,0,0.29197333256403607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,64,128,1,fp8,fp8,0,0.7229866981506348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,64,128,1,float16,float16,0,0.29971200227737427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,64,0,1,float16,float16,0,0.2972853382428487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,64,128,1,float16,fp8,0,0.29895466566085815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,64,128,1,fp8,fp8,0,0.7316799958546957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,64,0,1,float16,fp8,0,0.2958453297615051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,64,128,1,float16,float16,0,0.20585066080093384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,64,0,1,float16,float16,0,0.205402672290802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,64,0,1,fp8,fp8,0,0.7292799949645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,64,0,1,float16,float16,0,0.15079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,64,128,1,float16,fp8,0,0.1981546680132548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,64,128,1,fp8,fp8,0,0.40275732676188153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,64,0,1,float16,fp8,0,0.19775466124216715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,64,0,1,fp8,fp8,0,0.3999413251876831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,64,128,1,float16,float16,0,0.15075199802716574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,64,0,1,float16,fp8,0,0.15165332953135172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,64,128,1,float16,fp8,0,0.14923733472824097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,64,128,1,fp8,fp8,0,0.3731946547826131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,64,128,1,float16,float16,0,0.1513599952061971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,64,0,1,float16,float16,0,0.15050133069356283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,64,0,1,fp8,fp8,0,0.3732000192006429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,64,128,1,float16,fp8,0,0.1508853336175283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,64,128,1,fp8,fp8,0,0.37646400928497314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,64,0,1,float16,fp8,0,0.15119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,64,128,1,float16,fp8,0,0.1521013379096985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,64,0,1,fp8,fp8,0,0.37468798955281574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,64,128,1,float16,float16,0,0.15286933382352194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,64,0,1,float16,float16,0,0.1527359982331594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,64,0,1,float16,fp8,0,0.15266666809717813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,64,128,1,float16,fp8,0,0.15664000312487283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,64,128,1,fp8,fp8,0,0.37459198633829754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,64,128,1,float16,float16,0,0.1569386621316274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,64,128,1,fp8,fp8,0,0.3771040042241414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,64,0,1,fp8,fp8,0,0.37382400035858154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,64,0,1,float16,float16,0,0.15677332878112793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,64,0,1,float16,fp8,0,0.15593066811561584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,64,128,1,float16,float16,0,0.10977600018183391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,64,0,1,fp8,fp8,0,0.3773973385492961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,64,0,1,float16,float16,0,0.10946133732795715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,64,128,1,float16,fp8,0,0.10553600390752156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,64,128,1,fp8,fp8,0,0.21129600207010904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,64,0,1,float16,fp8,0,0.10572266578674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,64,0,1,fp8,fp8,0,0.21083199977874756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,64,128,1,float16,float16,0,0.08077866832415263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,64,0,1,float16,float16,0,0.08083733419577281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,64,128,1,float16,fp8,0,0.08072000245253245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,64,128,1,fp8,fp8,0,0.19684799512227377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,64,0,1,float16,fp8,0,0.08089600006739299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,64,0,1,fp8,fp8,0,0.197050670782725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,64,128,1,float16,float16,0,0.08109866579373677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,64,0,1,float16,float16,0,0.08158933122952779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,64,128,1,float16,fp8,0,0.08134933312733968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,64,128,1,fp8,fp8,0,0.1973759929339091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,64,0,1,float16,fp8,0,0.08065066734949748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,64,0,1,fp8,fp8,0,0.1963040033976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,64,0,1,float16,fp8,0,0.08163199822107951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,64,128,1,fp8,fp8,0,0.19643733898798624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,64,128,1,float16,float16,0,0.08147199948628743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,64,0,1,float16,float16,0,0.08105066418647766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,64,128,1,float16,fp8,0,0.08185600241025288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,64,0,1,fp8,fp8,0,0.1969333291053772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,64,128,1,float16,float16,0,0.08250133196512859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,64,0,1,float16,float16,0,0.08306133250395457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,64,128,1,float16,fp8,0,0.08290133376916249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,64,128,1,fp8,fp8,0,0.1990613341331482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,64,0,1,float16,fp8,0,0.08250133196512859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,64,0,1,fp8,fp8,0,0.2000160018603007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,64,128,1,float16,float16,0,0.05816533168156942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,64,0,1,float16,float16,0,0.057962665955225624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,64,128,1,float16,fp8,0,0.056261335810025535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,64,128,1,fp8,fp8,0,0.11614400148391724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,64,0,1,float16,fp8,0,0.056074668963750206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,64,0,1,fp8,fp8,0,0.11564800143241882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,64,0,1,fp8,fp8,0,0.10486400127410889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,64,128,1,float16,float16,0,0.04430399835109711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,64,128,1,float16,float16,0,0.04402133325735728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,64,0,1,float16,float16,0,0.04448533554871877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,64,128,1,float16,fp8,0,0.04409066836039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,64,128,1,fp8,fp8,0,0.10686399539311726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,64,0,1,float16,fp8,0,0.04399466514587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,64,0,1,float16,float16,0,0.04418666660785675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,64,128,1,float16,fp8,0,0.0444106658299764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,64,128,1,fp8,fp8,0,0.10531733433405559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,64,128,1,fp8,fp8,0,0.10689066847165425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,64,0,1,float16,fp8,0,0.0443146675825119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,64,0,1,fp8,fp8,0,0.10573866963386536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,64,128,1,float16,float16,0,0.044581333796183266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,64,0,1,float16,float16,0,0.044581333796183266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,64,128,1,float16,fp8,0,0.04429866870244344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,64,128,1,fp8,fp8,0,0.10626666744550069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,64,0,1,float16,fp8,0,0.04448533554871877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,64,0,1,fp8,fp8,0,0.10552533467610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,64,128,1,float16,float16,0,0.04529599845409393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,64,0,1,float16,float16,0,0.04494399825731913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,64,128,1,float16,fp8,0,0.04552533229192098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,64,0,1,float16,fp8,0,0.046112000942230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,64,128,1,float16,float16,0,0.03075733284155528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,64,0,1,fp8,fp8,0,0.10697600245475769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,64,0,1,float16,float16,0,0.030602666238943737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,64,128,1,float16,fp8,0,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,64,128,1,fp8,fp8,0,0.06749866902828217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,64,0,1,float16,fp8,0,0.03070399910211563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,64,0,1,fp8,fp8,0,0.06723199784755707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,64,128,1,float16,float16,0,0.029109333952267964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,64,0,1,float16,float16,0,0.02874133239189784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,64,128,1,float16,fp8,0,0.02888533224662145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,64,128,1,fp8,fp8,0,0.06298133234182994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,64,0,1,float16,fp8,0,0.028565332293510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,64,0,1,fp8,fp8,0,0.06249066690603892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,64,128,1,float16,float16,0,0.028581333657105763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,64,0,1,float16,float16,0,0.028778667251269024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,64,128,1,float16,fp8,0,0.029018667836983997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,64,128,1,fp8,fp8,0,0.061093335350354515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,64,0,1,float16,fp8,0,0.029120000700155895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,64,0,1,fp8,fp8,0,0.06328533093134563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,64,128,1,float16,float16,0,0.02935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,64,0,1,float16,float16,0,0.028565332293510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,64,128,1,float16,fp8,0,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,64,128,1,fp8,fp8,0,0.06267733375231425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,64,0,1,float16,fp8,0,0.02898666759332021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,64,0,1,fp8,fp8,0,0.06311999758084615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,64,128,1,float16,float16,0,0.02882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,64,0,1,float16,float16,0,0.029071999092896778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,64,128,1,float16,fp8,0,0.028959999481836956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,64,0,1,float16,float16,0,0.019920000185569126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,64,128,1,fp8,fp8,0,0.06351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,64,0,1,float16,fp8,0,0.02959466725587845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,64,0,1,fp8,fp8,0,0.06331199904282887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,64,128,1,float16,float16,0,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,64,128,1,float16,fp8,0,0.020224000016848247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,64,128,1,fp8,fp8,0,0.040149333576361336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,64,0,1,float16,fp8,0,0.020330666253964107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,64,0,1,fp8,fp8,0,0.040762667854626976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,64,128,1,float16,float16,0,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,64,0,1,fp8,fp8,0,0.03907199949026108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,64,0,1,float16,float16,0,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,64,128,1,float16,fp8,0,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,64,128,1,fp8,fp8,0,0.03878933439652125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,64,0,1,float16,fp8,0,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,64,128,1,float16,float16,0,0.019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,64,0,1,float16,float16,0,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,64,128,1,float16,float16,0,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,64,128,1,float16,fp8,0,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,64,128,1,fp8,fp8,0,0.03880000114440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,64,0,1,float16,fp8,0,0.019541333119074505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,64,0,1,float16,fp8,0,0.019482667247454327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,64,0,1,fp8,fp8,0,0.03843733419974645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,64,0,1,float16,float16,0,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,64,128,1,float16,fp8,0,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,64,128,1,fp8,fp8,0,0.03915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,64,0,1,fp8,fp8,0,0.03824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,64,128,1,float16,float16,0,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,64,0,1,float16,float16,0,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,64,128,1,float16,float16,0,0.014394666999578476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,64,128,1,float16,fp8,0,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,64,128,1,fp8,fp8,0,0.03957333415746689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,64,0,1,float16,fp8,0,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,64,0,1,fp8,fp8,0,0.03922666609287262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,64,0,1,float16,float16,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,64,128,1,float16,fp8,0,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,64,128,1,fp8,fp8,0,0.026672000686327618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,64,0,1,float16,fp8,0,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,64,0,1,fp8,fp8,0,0.02683199942111969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,64,128,1,float16,float16,0,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,64,0,1,float16,float16,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,64,128,1,float16,fp8,0,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,64,128,1,fp8,fp8,0,0.026389333109060924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,64,0,1,float16,fp8,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,64,0,1,fp8,fp8,0,0.02590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,64,128,1,float16,float16,0,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,64,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,64,128,1,float16,fp8,0,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,64,128,1,fp8,fp8,0,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,64,0,1,float16,fp8,0,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,64,0,1,fp8,fp8,0,0.026176000634829204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,64,128,1,float16,float16,0,0.014122666170199713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,64,0,1,float16,float16,0,0.013946666071812311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,64,128,1,float16,float16,0,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,64,128,1,float16,fp8,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,64,128,1,fp8,fp8,0,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,64,0,1,float16,fp8,0,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,64,0,1,fp8,fp8,0,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,64,0,1,float16,float16,0,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,64,128,1,float16,fp8,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,64,128,1,fp8,fp8,0,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,64,0,1,float16,fp8,0,0.014426667243242264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,64,128,1,float16,float16,0,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,64,0,1,fp8,fp8,0,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,64,0,1,float16,float16,0,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,64,128,1,float16,fp8,0,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,64,128,1,fp8,fp8,0,0.02232533444960912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,64,0,1,float16,fp8,0,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,64,0,1,fp8,fp8,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,64,128,1,float16,float16,0,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,64,0,1,float16,float16,0,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,64,128,1,float16,fp8,0,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,64,128,1,fp8,fp8,0,0.022170667846997578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,64,0,1,float16,fp8,0,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,64,0,1,fp8,fp8,0,0.021685334543387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,64,128,1,float16,float16,0,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,64,0,1,float16,float16,0,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,64,128,1,float16,fp8,0,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,64,128,1,fp8,fp8,0,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,64,0,1,float16,fp8,0,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,64,0,1,fp8,fp8,0,0.022064000368118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,64,128,1,float16,float16,0,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,64,0,1,float16,float16,0,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,64,128,1,float16,fp8,0,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,64,128,1,fp8,fp8,0,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,64,0,1,float16,fp8,0,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,64,0,1,fp8,fp8,0,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,64,128,1,float16,float16,0,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,64,0,1,float16,float16,0,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,64,128,1,float16,fp8,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,64,0,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,64,128,1,fp8,fp8,0,0.022330666581789654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,64,0,1,float16,fp8,0,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,64,0,1,fp8,fp8,0,0.022154666483402252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,64,128,1,float16,float16,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,64,128,1,float16,fp8,0,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,64,128,1,fp8,fp8,0,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,64,0,1,float16,fp8,0,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,64,0,1,fp8,fp8,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,64,128,1,float16,float16,0,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,64,0,1,float16,float16,0,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,64,128,1,float16,fp8,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,64,128,1,fp8,fp8,0,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,64,0,1,float16,fp8,0,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,64,0,1,fp8,fp8,0,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,64,0,1,float16,fp8,0,0.012341332932313284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,64,128,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,64,128,1,float16,fp8,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,64,128,1,fp8,fp8,0,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,64,0,1,fp8,fp8,0,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,64,128,1,float16,float16,0,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,64,0,1,float16,float16,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,64,128,1,float16,fp8,0,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,64,128,1,fp8,fp8,0,0.017946666727463405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,64,0,1,float16,fp8,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,64,0,1,fp8,fp8,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,64,128,1,float16,float16,0,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,64,128,1,float16,fp8,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,64,128,1,fp8,fp8,0,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,64,0,1,float16,fp8,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,64,0,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,64,128,1,float16,float16,0,0.20866666237513223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,64,0,1,float16,float16,0,0.206986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,64,128,1,float16,fp8,0,0.20724799235661825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,64,0,1,float16,fp8,0,0.20752533276875815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,64,128,1,fp8,fp8,0,0.6489973465601603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,64,128,1,float16,fp8,0,0.2088373303413391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,64,128,1,float16,float16,0,0.20752533276875815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,64,0,1,fp8,fp8,0,0.6504533290863037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,64,0,1,float16,float16,0,0.20784533023834229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,64,128,1,fp8,fp8,0,0.6490293343861898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,64,0,1,float16,fp8,0,0.20778133471806845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,64,0,1,float16,float16,0,0.20972265799840292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,64,128,1,float16,float16,0,0.20880534251530966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,64,128,1,float16,fp8,0,0.20834134022394815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,64,0,1,fp8,fp8,0,0.6508800188700358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,64,0,1,float16,fp8,0,0.20858667294184366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,64,128,1,float16,float16,0,0.21175465981165567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,64,128,1,fp8,fp8,0,0.650048017501831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,64,0,1,float16,float16,0,0.21110934019088745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,64,0,1,fp8,fp8,0,0.6482400099436442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,64,128,1,float16,fp8,0,0.21128000815709433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,64,0,1,float16,fp8,0,0.21171732743581137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,64,128,1,fp8,fp8,0,0.6527413527170817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,64,128,1,float16,float16,0,0.13124799728393555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,64,0,1,float16,float16,0,0.13064000010490417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,64,128,1,float16,fp8,0,0.12764267126719156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,64,0,1,fp8,fp8,0,0.6538186470667521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,64,0,1,float16,fp8,0,0.12706133723258972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,64,128,1,fp8,fp8,0,0.3486773173014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,64,128,1,float16,float16,0,0.10902399818102519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,64,0,1,float16,float16,0,0.10963199536005656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,64,128,1,fp8,fp8,0,0.33475200335184735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,64,0,1,fp8,fp8,0,0.35016000270843506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,64,128,1,float16,fp8,0,0.10872000455856323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,64,0,1,float16,fp8,0,0.10938133796056111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,64,128,1,float16,float16,0,0.10920533537864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,64,0,1,fp8,fp8,0,0.3359359900156657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,64,0,1,float16,float16,0,0.10928533474604289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,64,128,1,float16,fp8,0,0.10983999570210774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,64,128,1,fp8,fp8,0,0.3351840178171794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,64,0,1,float16,fp8,0,0.10949333508809407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,64,128,1,float16,float16,0,0.10990933577219646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,64,0,1,float16,float16,0,0.10922132929166158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,64,0,1,fp8,fp8,0,0.3344959815343221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,64,128,1,float16,fp8,0,0.11065066854159038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,64,0,1,float16,fp8,0,0.10998400052388509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,64,128,1,fp8,fp8,0,0.3349226713180542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,64,128,1,float16,float16,0,0.11141866445541382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,64,0,1,fp8,fp8,0,0.3359466791152954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,64,0,1,float16,float16,0,0.11114666859308879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,64,0,1,float16,float16,0,0.06858666737874348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,64,128,1,float16,fp8,0,0.11024000247319539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,64,0,1,float16,fp8,0,0.1108746627966563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,64,128,1,fp8,fp8,0,0.3370026747385661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,64,128,1,float16,float16,0,0.06764266888300578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,64,0,1,fp8,fp8,0,0.3375733296076457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,64,128,1,float16,fp8,0,0.0670773337284724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,64,128,1,float16,fp8,0,0.05836800237496694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,64,0,1,float16,fp8,0,0.06605333089828491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,64,128,1,fp8,fp8,0,0.18606932957967123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,64,0,1,fp8,fp8,0,0.1837760011355082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,64,128,1,float16,float16,0,0.05798399945100149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,64,0,1,float16,float16,0,0.05821333328882853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,64,128,1,fp8,fp8,0,0.1755626598993937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,64,0,1,float16,fp8,0,0.05825066566467285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,64,0,1,fp8,fp8,0,0.17571200927098593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,64,128,1,float16,float16,0,0.058090666929880776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,64,0,1,float16,float16,0,0.057962665955225624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,64,128,1,float16,fp8,0,0.058229332168896995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,64,0,1,float16,fp8,0,0.058431997895240784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,64,128,1,fp8,fp8,0,0.17606399456659952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,64,128,1,fp8,fp8,0,0.1760960022608439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,64,0,1,fp8,fp8,0,0.17468800147374472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,64,0,1,float16,fp8,0,0.0584746648867925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,64,128,1,float16,float16,0,0.058464000622431435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,64,0,1,fp8,fp8,0,0.17552000284194946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,64,0,1,float16,float16,0,0.058917333682378135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,64,128,1,float16,fp8,0,0.05820266902446747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,64,128,1,float16,float16,0,0.059024001161257424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,64,0,1,float16,float16,0,0.05937600135803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,64,128,1,float16,fp8,0,0.059114664793014526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,64,0,1,float16,fp8,0,0.05947199960549673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,64,128,1,fp8,fp8,0,0.17615999778111777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,64,0,1,fp8,fp8,0,0.1771679917971293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,64,128,1,float16,float16,0,0.03687999894221624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,64,0,1,float16,float16,0,0.03761066744724909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,64,128,1,float16,fp8,0,0.036042665441830955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,64,128,1,fp8,fp8,0,0.1014400025208791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,64,0,1,float16,fp8,0,0.03640533238649368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,64,0,1,fp8,fp8,0,0.10217066605885823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,64,128,1,float16,float16,0,0.03605866680542628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,64,0,1,float16,float16,0,0.036042665441830955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,64,128,1,float16,float16,0,0.03575466573238373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,64,128,1,float16,fp8,0,0.03611200054486593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,64,128,1,fp8,fp8,0,0.09753066301345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,64,0,1,float16,fp8,0,0.036271999279658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,64,0,1,fp8,fp8,0,0.09750933448473613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,64,0,1,float16,float16,0,0.036133334040641785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,64,128,1,float16,fp8,0,0.03618666778008143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,64,128,1,fp8,fp8,0,0.09681066870689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,64,0,1,float16,fp8,0,0.03618133316437403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,64,0,1,fp8,fp8,0,0.09745066364606221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,64,128,1,float16,float16,0,0.036271999279658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,64,0,1,float16,float16,0,0.03606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,64,128,1,float16,fp8,0,0.036277333895365395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,64,128,1,fp8,fp8,0,0.09745066364606221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,64,0,1,float16,fp8,0,0.03594133257865906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,64,0,1,fp8,fp8,0,0.09789333740870158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,64,128,1,float16,float16,0,0.03618666778008143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,64,0,1,fp8,fp8,0,0.09709866841634114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,64,128,1,float16,float16,0,0.02309333284695943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,64,0,1,float16,float16,0,0.03643733263015747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,64,128,1,float16,fp8,0,0.03623999903599421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,64,128,1,fp8,fp8,0,0.09784533580144246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,64,0,1,float16,fp8,0,0.03645866612593333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,64,128,1,float16,float16,0,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,64,0,1,float16,float16,0,0.023434666295846302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,64,128,1,float16,fp8,0,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,64,128,1,float16,fp8,0,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,64,128,1,fp8,fp8,0,0.059119999408721924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,64,0,1,float16,fp8,0,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,64,0,1,fp8,fp8,0,0.058335999647776283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,64,0,1,float16,float16,0,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,64,128,1,fp8,fp8,0,0.057674666245778404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,64,0,1,float16,fp8,0,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,64,0,1,fp8,fp8,0,0.056890666484832764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,64,0,1,fp8,fp8,0,0.05731200178464254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,64,128,1,float16,float16,0,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,64,0,1,float16,float16,0,0.022965334355831146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,64,128,1,float16,fp8,0,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,64,128,1,fp8,fp8,0,0.056608001391092934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,64,0,1,float16,fp8,0,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,64,128,1,float16,float16,0,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,64,0,1,float16,float16,0,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,64,128,1,float16,fp8,0,0.023728000621000927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,64,128,1,fp8,fp8,0,0.0572266678015391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,64,0,1,float16,fp8,0,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,64,0,1,fp8,fp8,0,0.057402665416399636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,64,128,1,float16,float16,0,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,64,0,1,float16,float16,0,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,64,128,1,float16,fp8,0,0.02309333284695943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,64,128,1,fp8,fp8,0,0.05675200124581655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,64,0,1,float16,fp8,0,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,64,0,1,fp8,fp8,0,0.057071998715400696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,64,128,1,float16,float16,0,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,64,0,1,float16,float16,0,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,64,128,1,float16,fp8,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,64,128,1,fp8,fp8,0,0.03602666656176249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,64,0,1,float16,fp8,0,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,64,0,1,fp8,fp8,0,0.03572800010442734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,64,128,1,float16,float16,0,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,64,0,1,float16,float16,0,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,64,128,1,float16,fp8,0,0.015664000064134598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,64,128,1,fp8,fp8,0,0.03526933242877325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,64,0,1,float16,fp8,0,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,64,0,1,fp8,fp8,0,0.03545066714286804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,64,128,1,float16,float16,0,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,64,0,1,float16,float16,0,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,64,128,1,float16,fp8,0,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,64,128,1,float16,fp8,0,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,64,128,1,fp8,fp8,0,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,64,0,1,float16,fp8,0,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,64,0,1,fp8,fp8,0,0.0352906659245491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,64,128,1,float16,float16,0,0.015685333559910457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,64,0,1,float16,float16,0,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,64,128,1,fp8,fp8,0,0.03513599932193756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,64,0,1,float16,fp8,0,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,64,128,1,fp8,fp8,0,0.03566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,64,0,1,fp8,fp8,0,0.035029334326585136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,64,128,1,float16,float16,0,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,64,0,1,float16,float16,0,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,64,128,1,float16,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,64,0,1,float16,fp8,0,0.015978666643301647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,64,0,1,fp8,fp8,0,0.03527999917666117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,64,128,1,float16,float16,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,64,0,1,float16,float16,0,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,64,128,1,float16,fp8,0,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,64,128,1,fp8,fp8,0,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,64,0,1,float16,fp8,0,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,64,0,1,fp8,fp8,0,0.026464000344276428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,64,128,1,float16,float16,0,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,64,0,1,float16,float16,0,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,64,128,1,float16,fp8,0,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,64,128,1,fp8,fp8,0,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,64,0,1,float16,fp8,0,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,64,0,1,fp8,fp8,0,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,64,0,1,float16,fp8,0,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,64,128,1,float16,float16,0,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,64,0,1,float16,float16,0,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,64,128,1,float16,fp8,0,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,64,128,1,fp8,fp8,0,0.02550933261712392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,64,0,1,fp8,fp8,0,0.02518933266401291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,64,128,1,float16,float16,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,64,0,1,float16,float16,0,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,64,128,1,float16,fp8,0,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,64,128,1,fp8,fp8,0,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,64,0,1,float16,fp8,0,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,64,0,1,fp8,fp8,0,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,64,128,1,float16,float16,0,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,64,0,1,float16,float16,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,64,128,1,float16,fp8,0,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,64,128,1,fp8,fp8,0,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,64,128,1,fp8,fp8,0,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,64,0,1,float16,fp8,0,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,64,0,1,fp8,fp8,0,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,64,128,1,float16,float16,0,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,64,0,1,float16,float16,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,64,128,1,float16,fp8,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,64,0,1,float16,fp8,0,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,64,0,1,fp8,fp8,0,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,64,128,1,float16,float16,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,64,0,1,float16,float16,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,64,128,1,float16,fp8,0,0.012362666428089142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,64,128,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,64,0,1,float16,fp8,0,0.012234666695197424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,64,0,1,fp8,fp8,0,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,64,128,1,float16,float16,0,0.012117333710193634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,64,0,1,float16,float16,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,64,128,1,float16,fp8,0,0.012272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,64,128,1,fp8,fp8,0,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,64,0,1,float16,fp8,0,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,64,0,1,fp8,fp8,0,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,64,128,1,float16,float16,0,0.012122667084137598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,64,0,1,float16,float16,0,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,64,128,1,float16,fp8,0,0.012506666282812754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,64,128,1,fp8,fp8,0,0.02199999988079071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,64,0,1,float16,fp8,0,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,64,0,1,fp8,fp8,0,0.02146666745344798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,64,128,1,float16,float16,0,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,64,0,1,float16,float16,0,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,64,128,1,float16,fp8,0,0.012527999778588613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,64,128,1,fp8,fp8,0,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,64,0,1,float16,fp8,0,0.012319999436537424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,64,0,1,fp8,fp8,0,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,64,128,1,float16,float16,0,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,64,0,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,64,128,1,float16,fp8,0,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,64,128,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,64,0,1,float16,fp8,0,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,64,0,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,64,128,1,float16,float16,0,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,64,0,1,float16,float16,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,64,128,1,float16,fp8,0,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,64,128,1,fp8,fp8,0,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,64,0,1,float16,fp8,0,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,64,0,1,fp8,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,64,128,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,64,0,1,float16,float16,0,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,64,128,1,float16,fp8,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,64,128,1,float16,fp8,0,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,64,128,1,fp8,fp8,0,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,64,0,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,64,0,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,64,0,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,64,128,1,float16,float16,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,64,0,1,float16,float16,0,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,64,128,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,64,0,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,64,128,1,float16,float16,0,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,64,0,1,float16,float16,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,64,128,1,float16,fp8,0,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,64,128,1,fp8,fp8,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,64,0,1,float16,fp8,0,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,64,0,1,fp8,fp8,0,0.017893332988023758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,64,128,1,float16,float16,0,5.745424270629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,64,128,1,float16,fp8,0,5.702431996663411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,64,128,1,float16,float16,0,5.984352111816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,64,128,1,fp8,fp8,0,7.347279866536458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,64,128,1,float16,fp8,0,5.877424240112305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,64,128,1,fp8,fp8,0,7.512575785319011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,64,128,1,float16,float16,0,5.943183898925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,64,128,1,float16,fp8,0,5.902693430582683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,64,128,1,fp8,fp8,0,7.52998415629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,64,0,1,fp8,fp8,0,35.98240407307943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,64,0,1,float16,float16,0,43.74058532714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,64,0,1,float16,fp8,0,42.31168111165365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,64,0,1,fp8,fp8,0,35.104390462239586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,64,0,1,float16,float16,0,40.57461293538412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,64,128,1,float16,float16,0,6.045386632283528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,64,0,1,float16,fp8,0,41.854164123535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,64,128,1,float16,fp8,0,6.135178883870442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,64,128,1,fp8,fp8,0,7.5595143636067705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,64,0,1,float16,float16,0,41.14453887939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,64,128,1,float16,float16,0,3.4097652435302734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,64,128,1,float16,fp8,0,3.3476905822753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,64,128,1,fp8,fp8,0,4.266778628031413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,64,0,1,fp8,fp8,0,35.4056650797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,64,0,1,float16,float16,0,19.325642903645832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,64,128,1,float16,float16,0,2.840991973876953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,64,0,1,float16,fp8,0,41.319567362467446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,64,128,1,float16,fp8,0,2.8007628122965493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,64,128,1,fp8,fp8,0,3.7073227564493814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,64,0,1,fp8,fp8,0,17.732101440429688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,64,0,1,float16,float16,0,42.63463592529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,64,0,1,float16,fp8,0,19.265797932942707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,64,0,1,fp8,fp8,0,37.12768046061198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,64,128,1,float16,float16,0,2.9330825805664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,64,128,1,float16,fp8,0,2.937199910481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,64,0,1,float16,fp8,0,41.25414276123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,64,128,1,fp8,fp8,0,3.819642702738444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,64,128,1,float16,float16,0,2.9699201583862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,64,128,1,float16,fp8,0,2.948330561319987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,64,0,1,float16,float16,0,19.46619160970052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,64,0,1,fp8,fp8,0,17.139119466145832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,64,0,1,float16,fp8,0,18.80848566691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,64,128,1,fp8,fp8,0,3.8474400838216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,64,0,1,float16,float16,0,18.970704396565754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,64,128,1,float16,float16,0,2.9978720347086587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,64,0,1,fp8,fp8,0,17.272171020507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,64,0,1,float16,fp8,0,19.40332794189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,64,128,1,float16,fp8,0,2.9701760609944663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,64,128,1,fp8,fp8,0,3.8717174530029297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,64,128,1,float16,float16,0,1.67575470606486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,64,0,1,float16,float16,0,19.019210815429688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,64,128,1,float16,fp8,0,1.6612106959025066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,64,128,1,fp8,fp8,0,2.1441920598347983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,64,0,1,float16,fp8,0,18.809141794840496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,64,0,1,fp8,fp8,0,18.746480305989582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,64,128,1,float16,float16,0,1.5108213424682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,64,0,1,float16,float16,0,9.79089609781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,64,128,1,float16,fp8,0,1.4830187161763508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,64,0,1,float16,float16,0,19.2445551554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,64,0,1,fp8,fp8,0,8.99995231628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,64,128,1,fp8,fp8,0,1.938421408335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,64,0,1,float16,fp8,0,9.797082901000977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,64,0,1,fp8,fp8,0,17.303674062093098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,64,128,1,float16,float16,0,1.5219626426696777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,64,128,1,float16,fp8,0,1.4924480120340984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,64,0,1,float16,fp8,0,20.28112030029297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,64,128,1,fp8,fp8,0,1.9429599444071453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,64,128,1,float16,float16,0,1.5467573801676433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,64,0,1,float16,float16,0,9.609866460164389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,64,128,1,float16,fp8,0,1.4913867314656575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,64,128,1,fp8,fp8,0,1.9447253545125325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,64,0,1,fp8,fp8,0,9.014570871988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,64,0,1,float16,fp8,0,10.590272267659506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,64,0,1,float16,float16,0,9.523429234822592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,64,128,1,float16,float16,0,1.5345226923624675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,64,0,1,fp8,fp8,0,8.670581181844076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,64,0,1,float16,fp8,0,9.547445297241211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,64,128,1,float16,fp8,0,1.5092426935831706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,64,128,1,fp8,fp8,0,1.9650880495707195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,64,128,1,float16,float16,0,1.0707733631134033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,64,0,1,float16,float16,0,9.697925567626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,64,0,1,fp8,fp8,0,8.706031799316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,64,0,1,float16,fp8,0,9.638026555379232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,64,128,1,float16,fp8,0,1.0714133580525715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,64,128,1,fp8,fp8,0,1.2879093488057454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,64,128,1,float16,float16,0,1.0705599784851074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,64,0,1,float16,float16,0,9.903039932250977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,64,0,1,float16,float16,0,5.778048197428386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,64,128,1,float16,fp8,0,1.0715359846750896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,64,0,1,fp8,fp8,0,8.720149358113607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,64,128,1,fp8,fp8,0,1.2875733375549316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,64,0,1,float16,fp8,0,9.978309631347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,64,0,1,fp8,fp8,0,4.7013654708862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,64,0,1,float16,fp8,0,5.172176043192546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,64,128,1,float16,float16,0,1.0700213114420574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,64,128,1,float16,fp8,0,1.0712107022603352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,64,128,1,fp8,fp8,0,1.2784533500671387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,64,0,1,float16,float16,0,5.327199935913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,64,0,1,float16,fp8,0,5.387733459472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,64,128,1,float16,float16,0,1.0828533172607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,64,0,1,fp8,fp8,0,4.6601918538411455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,64,128,1,float16,fp8,0,1.081653356552124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,64,0,1,float16,float16,0,5.222410519917806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,64,128,1,fp8,fp8,0,1.2981332937876384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,64,0,1,float16,fp8,0,5.168293317159017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,64,0,1,fp8,fp8,0,4.6667680740356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,64,128,1,float16,float16,0,1.0707626342773438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,64,128,1,float16,fp8,0,1.0869920253753662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,64,0,1,float16,float16,0,5.227967898050944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,64,128,1,fp8,fp8,0,1.283962647120158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,64,0,1,float16,fp8,0,5.112309455871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,64,0,1,fp8,fp8,0,4.686896006266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,64,0,1,float16,float16,0,5.1034345626831055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,64,0,1,float16,fp8,0,5.210261344909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,64,128,1,float16,float16,0,4.289552052815755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,64,0,1,fp8,fp8,0,4.693845431009929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,64,128,1,float16,fp8,0,4.249135971069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,64,128,1,fp8,fp8,0,5.480010350545247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,64,128,1,float16,float16,0,4.481722513834636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,64,128,1,float16,fp8,0,4.469632148742676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,64,128,1,fp8,fp8,0,5.631466547648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,64,128,1,float16,float16,0,4.498490651448567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,64,128,1,float16,fp8,0,4.446154594421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,64,0,1,float16,float16,0,22.11218007405599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,64,0,1,fp8,fp8,0,20.106292724609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,64,0,1,float16,fp8,0,21.98962148030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,64,128,1,fp8,fp8,0,5.636000315348308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,64,0,1,float16,float16,0,22.766794840494793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,64,0,1,fp8,fp8,0,20.562299092610676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,64,128,1,float16,float16,0,4.496298789978027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,64,0,1,float16,fp8,0,22.551556905110676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,64,128,1,float16,fp8,0,4.488320032755534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,64,0,1,float16,float16,0,22.253130594889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,64,128,1,fp8,fp8,0,5.702250798543294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,64,128,1,float16,float16,0,2.5158507029215493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,64,128,1,float16,fp8,0,2.4458667437235513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,64,128,1,fp8,fp8,0,3.190890630086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,64,0,1,fp8,fp8,0,20.233605702718098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,64,0,1,float16,fp8,0,22.37451680501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,64,128,1,float16,float16,0,2.092672030131022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,64,0,1,float16,float16,0,12.606522878011068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,64,0,1,float16,float16,0,22.018773396809895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,64,128,1,float16,fp8,0,2.074069341023763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,64,0,1,float16,fp8,0,11.423973083496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,64,0,1,fp8,fp8,0,10.447002410888672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,64,128,1,fp8,fp8,0,2.7501653035481772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,64,0,1,fp8,fp8,0,20.191307067871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,64,128,1,float16,float16,0,2.139253298441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,64,0,1,float16,fp8,0,22.377339680989582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,64,128,1,float16,fp8,0,2.1084906260172525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,64,128,1,fp8,fp8,0,2.7983840306599936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,64,128,1,float16,float16,0,2.1443360646565757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,64,0,1,float16,float16,0,11.285349527994791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,64,128,1,float16,fp8,0,2.114837328592936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,64,0,1,fp8,fp8,0,9.968373616536459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,64,0,1,float16,fp8,0,12.077205657958984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,64,128,1,fp8,fp8,0,2.8694985707600913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,64,0,1,float16,float16,0,11.445743560791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,64,128,1,float16,float16,0,2.174704074859619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,64,0,1,fp8,fp8,0,10.03659756978353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,64,0,1,float16,fp8,0,11.029034932454428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,64,128,1,float16,fp8,0,2.1472907066345215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,64,128,1,fp8,fp8,0,2.84440549214681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,64,0,1,float16,float16,0,11.203664143880209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,64,128,1,float16,float16,0,1.259173313776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,64,128,1,float16,fp8,0,1.2565226554870605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,64,128,1,fp8,fp8,0,1.6140586535135906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,64,0,1,fp8,fp8,0,10.04586664835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,64,0,1,float16,fp8,0,10.94918950398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,64,128,1,float16,float16,0,1.1297600269317627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,64,0,1,float16,float16,0,5.919424057006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,64,0,1,float16,float16,0,11.101130167643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,64,128,1,float16,fp8,0,1.117466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,64,128,1,fp8,fp8,0,1.4700746536254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,64,0,1,float16,fp8,0,5.895802815755208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,64,0,1,fp8,fp8,0,5.40123176574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,64,0,1,fp8,fp8,0,10.435258865356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,64,0,1,float16,fp8,0,11.489264170328775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,64,128,1,float16,float16,0,1.147104024887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,64,128,1,float16,fp8,0,1.1269546349843342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,64,0,1,float16,float16,0,5.812143961588542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,64,128,1,fp8,fp8,0,1.458074728647868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,64,0,1,fp8,fp8,0,5.096000035603841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,64,128,1,float16,float16,0,1.1370399792989094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,64,0,1,float16,fp8,0,5.903242746988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,64,128,1,float16,fp8,0,1.129647970199585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,64,128,1,fp8,fp8,0,1.4592906634012859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,64,0,1,float16,float16,0,5.986448287963867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,64,128,1,float16,float16,0,1.1489066282908122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,64,0,1,fp8,fp8,0,5.106282552083333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,64,0,1,float16,fp8,0,5.616186777750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,64,128,1,float16,fp8,0,1.1419359842936199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,64,0,1,float16,float16,0,5.7706349690755205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,64,128,1,fp8,fp8,0,1.477952003479004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,64,0,1,fp8,fp8,0,5.093450546264648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,64,0,1,float16,fp8,0,5.822698593139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,64,128,1,float16,float16,0,0.8100053469340006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,64,128,1,float16,fp8,0,0.8105066617329916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,64,128,1,fp8,fp8,0,0.9711146354675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,64,0,1,float16,float16,0,5.973621368408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,64,0,1,float16,float16,0,3.0374345779418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,64,128,1,float16,float16,0,0.8072640101114908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,64,0,1,float16,fp8,0,5.859872182210286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,64,0,1,float16,fp8,0,3.0390774408976235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,64,128,1,float16,fp8,0,0.8200000127156576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,64,0,1,fp8,fp8,0,5.126202583312988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,64,0,1,fp8,fp8,0,2.8024959564208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,64,128,1,fp8,fp8,0,0.9700586795806885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,64,0,1,float16,float16,0,3.040538787841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,64,128,1,float16,float16,0,0.8195199966430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,64,128,1,float16,fp8,0,0.8077387015024821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,64,128,1,fp8,fp8,0,0.9677920341491699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,64,128,1,float16,float16,0,0.8065919876098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,64,0,1,fp8,fp8,0,2.8139146169026694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,64,0,1,float16,fp8,0,3.044095993041992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,64,0,1,float16,float16,0,3.047674814860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,64,128,1,float16,fp8,0,0.8071200052897135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,64,0,1,fp8,fp8,0,2.857802708943685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,64,0,1,float16,fp8,0,3.0435358683268228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,64,128,1,fp8,fp8,0,0.9753386974334717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,64,128,1,float16,float16,0,0.8065226872762045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,64,0,1,float16,float16,0,3.1028213500976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,64,128,1,float16,fp8,0,0.8101812998453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,64,128,1,fp8,fp8,0,0.9723573525746664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,64,0,1,float16,fp8,0,3.110015869140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,64,0,1,fp8,fp8,0,2.8555466334025064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,64,0,1,float16,float16,0,3.042869249979655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,64,0,1,float16,fp8,0,3.117157300313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,64,0,1,fp8,fp8,0,2.8493385314941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,64,128,1,float16,float16,0,3.489642779032389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,64,128,1,float16,fp8,0,3.4882081349690757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,64,128,1,fp8,fp8,0,4.564101219177246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,64,128,1,float16,float16,0,3.7250026067097983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,64,128,1,float16,fp8,0,3.686880111694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,64,128,1,fp8,fp8,0,4.741312026977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,64,128,1,float16,float16,0,3.716773351033529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,64,128,1,float16,fp8,0,3.694133440653483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,64,0,1,fp8,fp8,0,14.2215576171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,64,0,1,float16,float16,0,15.616959889729818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,64,0,1,float16,fp8,0,15.47646967569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,64,0,1,float16,float16,0,15.854036966959635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,64,0,1,fp8,fp8,0,14.368816375732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,64,128,1,fp8,fp8,0,4.720362663269043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,64,128,1,float16,float16,0,3.7488266626993814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,64,0,1,float16,fp8,0,15.848138173421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,64,0,1,float16,float16,0,15.696367899576822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,64,128,1,float16,fp8,0,3.7190240224202475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,64,128,1,fp8,fp8,0,4.734570821126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,64,128,1,float16,float16,0,2.0799946784973145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,64,128,1,float16,fp8,0,1.992517312367757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,64,128,1,fp8,fp8,0,2.647712071736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,64,0,1,fp8,fp8,0,14.565781911214193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,64,0,1,float16,fp8,0,15.990895589192709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,64,128,1,float16,float16,0,1.75601593653361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,64,0,1,float16,float16,0,8.651685078938803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,64,0,1,float16,float16,0,17.428394317626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,64,128,1,float16,fp8,0,1.71723206837972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,64,128,1,fp8,fp8,0,2.2936479250590005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,64,0,1,fp8,fp8,0,14.793866475423178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,64,0,1,float16,fp8,0,8.07095464070638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,64,0,1,fp8,fp8,0,7.46232541402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,64,0,1,float16,fp8,0,16.60040028889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,64,128,1,float16,float16,0,1.7672212918599446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,64,128,1,float16,fp8,0,1.7430933316548665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,64,0,1,float16,float16,0,8.125690460205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,64,128,1,fp8,fp8,0,2.2895572980244956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,64,0,1,fp8,fp8,0,7.126768112182617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,64,0,1,float16,fp8,0,7.957541147867839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,64,128,1,float16,float16,0,1.7731946309407551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,64,128,1,float16,fp8,0,1.7477067311604817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,64,128,1,fp8,fp8,0,2.3043786684672036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,64,0,1,float16,float16,0,7.8779042561848955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,64,128,1,float16,float16,0,1.789621353149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,64,0,1,float16,fp8,0,7.7820479075113935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,64,0,1,fp8,fp8,0,7.141509373982747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,64,128,1,float16,fp8,0,1.7662933667500813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,64,128,1,fp8,fp8,0,2.3416800498962402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,64,128,1,float16,float16,0,1.0427573521931965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,64,0,1,float16,float16,0,7.863712310791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,64,0,1,fp8,fp8,0,7.153760274251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,64,0,1,float16,fp8,0,7.900735855102539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,64,128,1,float16,fp8,0,1.044986645380656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,64,128,1,fp8,fp8,0,1.3458186785380046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,64,0,1,float16,float16,0,7.90715217590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,64,0,1,float16,float16,0,4.209967931111653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,64,128,1,float16,float16,0,0.9540320237477621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,64,128,1,float16,fp8,0,0.943674643834432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,64,0,1,float16,fp8,0,4.237375895182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,64,128,1,fp8,fp8,0,1.2302772998809814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,64,0,1,fp8,fp8,0,7.179226557413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,64,0,1,fp8,fp8,0,3.781130790710449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,64,0,1,float16,fp8,0,7.920986811319987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,64,128,1,float16,float16,0,0.9866133530934652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,64,128,1,float16,fp8,0,0.9387093385060629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,64,0,1,float16,float16,0,4.318330764770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,64,128,1,fp8,fp8,0,1.2239306767781575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,64,0,1,float16,fp8,0,4.153850555419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,64,128,1,float16,float16,0,0.9505600134531657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,64,0,1,fp8,fp8,0,3.6618293126424155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,64,128,1,float16,fp8,0,0.9425973097483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,64,0,1,float16,float16,0,4.207920074462891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,64,128,1,fp8,fp8,0,1.2297173341115315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,64,0,1,fp8,fp8,0,3.653045336405436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,64,0,1,float16,fp8,0,4.167200088500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,64,128,1,float16,float16,0,0.9865547021230062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,64,128,1,float16,fp8,0,0.9474506378173828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,64,0,1,float16,float16,0,4.11083189646403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,64,128,1,fp8,fp8,0,1.2792960007985432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,64,128,1,float16,float16,0,0.6805386543273926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,64,0,1,fp8,fp8,0,3.6607945760091147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,64,0,1,float16,fp8,0,4.226389249165853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,64,128,1,float16,fp8,0,0.6802933216094971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,64,128,1,fp8,fp8,0,0.8192959626515707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,64,0,1,float16,float16,0,4.261904080708821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,64,0,1,float16,float16,0,2.207109292348226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,64,128,1,float16,float16,0,0.6770933469136556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,64,0,1,fp8,fp8,0,3.6698452631632485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,64,0,1,float16,fp8,0,4.087199846903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,64,0,1,float16,fp8,0,2.2111199696858725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,64,0,1,fp8,fp8,0,2.0559093157450357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,64,128,1,float16,fp8,0,0.6818079948425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,64,128,1,fp8,fp8,0,0.8153493404388428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,64,128,1,float16,float16,0,0.6802453200022379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,64,0,1,float16,float16,0,2.2102346420288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,64,128,1,float16,fp8,0,0.6821280320485433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,64,0,1,float16,fp8,0,2.2838187217712402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,64,0,1,fp8,fp8,0,2.0527092615763345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,64,128,1,fp8,fp8,0,0.8159466584523519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,64,0,1,float16,float16,0,2.2128426233927407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,64,128,1,float16,float16,0,0.6843626499176025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,64,128,1,float16,fp8,0,0.6790560086568197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,64,0,1,float16,fp8,0,2.209935983022054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,64,128,1,fp8,fp8,0,0.8258079687754313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,64,0,1,fp8,fp8,0,2.051541328430176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,64,0,1,float16,float16,0,2.210111935933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,64,128,1,float16,float16,0,0.6821066538492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,64,0,1,float16,fp8,0,2.2125280698140464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,64,128,1,float16,fp8,0,0.6797440052032471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,64,0,1,fp8,fp8,0,2.088165283203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,64,128,1,fp8,fp8,0,0.8216586907704672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,64,0,1,float16,float16,0,2.213637351989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,64,0,1,fp8,fp8,0,2.053445339202881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,64,0,1,float16,fp8,0,2.2191947301228843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,64,128,1,float16,float16,0,5.686373392740886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,64,128,1,float16,fp8,0,5.6434987386067705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,64,128,1,fp8,fp8,0,7.220298767089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,64,128,1,float16,float16,0,5.8986663818359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,64,128,1,float16,fp8,0,5.875712076822917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,64,128,1,fp8,fp8,0,7.427370707194011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,64,128,1,float16,float16,0,5.898837407430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,64,0,1,fp8,fp8,0,18.859364827473957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,64,0,1,float16,float16,0,20.845706939697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,64,0,1,float16,fp8,0,20.67733383178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,64,0,1,float16,float16,0,21.075397491455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,64,128,1,float16,fp8,0,5.8251997629801435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,64,0,1,fp8,fp8,0,19.188714345296223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,64,128,1,fp8,fp8,0,7.441509246826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,64,0,1,float16,fp8,0,20.989434560139973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,64,128,1,float16,float16,0,6.044623692830403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,64,128,1,float16,fp8,0,5.948618570963542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,64,128,1,fp8,fp8,0,7.499488194783528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,64,0,1,float16,float16,0,21.21389389038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,64,128,1,float16,float16,0,3.3697598775227866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,64,128,1,float16,fp8,0,3.3365227381388345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,64,0,1,fp8,fp8,0,19.58845392862956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,64,128,1,fp8,fp8,0,4.195551872253418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,64,0,1,float16,fp8,0,21.343658447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,64,128,1,float16,float16,0,2.7296107610066733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,64,0,1,float16,float16,0,22.233194986979168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,64,0,1,float16,float16,0,10.862794240315756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,64,128,1,float16,fp8,0,2.690810521443685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,64,0,1,fp8,fp8,0,19.18389892578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,64,128,1,fp8,fp8,0,3.604677200317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,64,0,1,float16,fp8,0,11.265706380208334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,64,0,1,fp8,fp8,0,10.032389322916666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,64,0,1,float16,fp8,0,21.489339192708332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,64,128,1,float16,float16,0,2.905861218770345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,64,0,1,float16,float16,0,10.448783874511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,64,128,1,float16,fp8,0,2.8887306849161782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,64,128,1,fp8,fp8,0,3.733920097351074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,64,0,1,float16,fp8,0,10.241141637166342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,64,128,1,float16,float16,0,2.9068800608317056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,64,0,1,fp8,fp8,0,9.420879999796549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,64,128,1,float16,fp8,0,2.8675146102905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,64,128,1,fp8,fp8,0,3.760469436645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,64,0,1,float16,float16,0,10.474106470743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,64,0,1,fp8,fp8,0,9.596944173177084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,64,128,1,float16,float16,0,2.915664037068685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,64,0,1,float16,fp8,0,10.583082834879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,64,128,1,float16,fp8,0,2.915264129638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,64,0,1,float16,float16,0,10.423327763875326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,64,128,1,fp8,fp8,0,3.7836694717407227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,64,128,1,float16,float16,0,1.6326932907104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,64,0,1,fp8,fp8,0,9.591338475545248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,64,128,1,float16,fp8,0,1.5844319661458333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,64,0,1,float16,fp8,0,11.327935536702475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,64,128,1,fp8,fp8,0,2.061936060587565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,64,128,1,float16,float16,0,1.4117706616719563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,64,0,1,float16,float16,0,5.455322901407878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,64,0,1,float16,float16,0,10.793375651041666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,64,0,1,fp8,fp8,0,10.495375951131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,64,128,1,float16,fp8,0,1.394543965657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,64,0,1,float16,fp8,0,10.787525177001953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,64,0,1,fp8,fp8,0,5.083349227905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,64,0,1,float16,fp8,0,5.992581049601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,64,128,1,fp8,fp8,0,1.8285333315531414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,64,128,1,float16,float16,0,1.4185439745585124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,64,128,1,float16,fp8,0,1.5087520281473796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,64,0,1,float16,float16,0,5.635679880777995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,64,128,1,fp8,fp8,0,1.8273812929789226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,64,128,1,float16,float16,0,1.4625546137491863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,64,0,1,fp8,fp8,0,4.789258639017741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,64,0,1,float16,fp8,0,5.587797164916992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,64,0,1,float16,float16,0,5.338837305704753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,64,128,1,float16,fp8,0,1.4056746164957683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,64,0,1,fp8,fp8,0,4.770448048909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,64,128,1,fp8,fp8,0,1.8512959480285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,64,128,1,float16,float16,0,1.4343040784200032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,64,0,1,float16,fp8,0,5.382207870483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,64,0,1,float16,float16,0,5.29696528116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,64,128,1,float16,fp8,0,1.4405546188354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,64,128,1,fp8,fp8,0,1.8624587059020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,64,128,1,float16,float16,0,0.8469066619873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,64,0,1,fp8,fp8,0,4.764447848002116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,64,0,1,float16,fp8,0,5.727231979370117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,64,128,1,float16,fp8,0,0.8384906450907389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,64,128,1,fp8,fp8,0,1.0860319932301838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,64,0,1,float16,float16,0,5.465066909790039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,64,128,1,float16,float16,0,0.7617599964141846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,64,0,1,float16,float16,0,2.757557233174642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,64,128,1,float16,fp8,0,0.7791679700215658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,64,0,1,float16,fp8,0,2.8675788243611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,64,128,1,fp8,fp8,0,0.9801013469696045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,64,0,1,float16,fp8,0,5.2617387771606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,64,0,1,fp8,fp8,0,2.5614026387532554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,64,0,1,fp8,fp8,0,4.770037333170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,64,128,1,float16,float16,0,0.7933493455251058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,64,0,1,float16,float16,0,2.659722646077474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,64,128,1,float16,fp8,0,0.7655466397603353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,64,128,1,fp8,fp8,0,0.9938027064005533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,64,0,1,float16,fp8,0,2.7259254455566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,64,128,1,float16,float16,0,0.8024106820424398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,64,0,1,fp8,fp8,0,2.4610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,64,128,1,float16,fp8,0,0.764234701792399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,64,0,1,float16,float16,0,2.7526985804239907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,64,128,1,fp8,fp8,0,1.0161759853363037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,64,0,1,float16,fp8,0,2.6559093793233237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,64,128,1,float16,float16,0,0.7764693101247152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,64,0,1,fp8,fp8,0,2.4661760330200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,64,128,1,float16,fp8,0,0.7689279715220133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,64,0,1,float16,float16,0,2.8164800008138022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,64,128,1,fp8,fp8,0,0.9975466728210449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,64,0,1,fp8,fp8,0,2.4569066365559897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,64,128,1,float16,float16,0,0.5491626660029093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,64,0,1,float16,fp8,0,2.725306510925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,64,128,1,float16,fp8,0,0.550378680229187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,64,0,1,float16,float16,0,2.6738293965657554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,64,0,1,float16,float16,0,1.5209439595540364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,64,128,1,fp8,fp8,0,0.661349336306254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,64,0,1,float16,fp8,0,2.7137654622395835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,64,0,1,fp8,fp8,0,2.4726346333821616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,64,128,1,float16,float16,0,0.5465706586837769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,64,128,1,float16,fp8,0,0.548799991607666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,64,0,1,float16,fp8,0,1.50874662399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,64,0,1,fp8,fp8,0,1.4116106033325195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,64,128,1,fp8,fp8,0,0.6598026752471924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,64,0,1,float16,float16,0,1.5116000175476074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,64,128,1,float16,float16,0,0.5523413419723511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,64,0,1,float16,fp8,0,1.5702719688415527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,64,0,1,fp8,fp8,0,1.4097867012023926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,64,128,1,float16,fp8,0,0.5480213165283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,64,128,1,fp8,fp8,0,0.6619786818822225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,64,0,1,float16,float16,0,1.5118773778279622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,64,128,1,float16,float16,0,0.5521920124689738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,64,0,1,float16,fp8,0,1.5117866198221843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,64,0,1,fp8,fp8,0,1.411695957183838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,64,128,1,float16,fp8,0,0.5482720136642456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,64,128,1,fp8,fp8,0,0.663541316986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,64,0,1,float16,float16,0,1.5130079587300618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,64,128,1,float16,fp8,0,0.5489013195037842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,64,128,1,float16,float16,0,0.5528159936269125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,64,0,1,float16,fp8,0,1.5138986905415852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,64,0,1,fp8,fp8,0,1.4095573425292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,64,128,1,fp8,fp8,0,0.6613333225250244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,64,0,1,float16,float16,0,1.512719949086507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,64,0,1,float16,fp8,0,1.5138932863871257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,64,0,1,fp8,fp8,0,1.410746733347575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,64,128,1,float16,float16,0,4.247930526733398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,64,128,1,float16,fp8,0,4.207482655843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,64,128,1,float16,float16,0,4.44379202524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,64,128,1,fp8,fp8,0,5.4469655354817705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,64,128,1,float16,fp8,0,4.380629221598308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,64,128,1,fp8,fp8,0,5.536693572998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,64,0,1,float16,float16,0,12.344341278076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,64,0,1,fp8,fp8,0,11.293621063232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,64,128,1,float16,float16,0,4.417440096537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,64,0,1,float16,fp8,0,12.30514653523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,64,0,1,float16,float16,0,12.715508778889975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,64,128,1,float16,fp8,0,4.417637189229329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,64,0,1,fp8,fp8,0,11.481338500976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,64,128,1,fp8,fp8,0,5.635215759277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,64,0,1,float16,fp8,0,12.508218129475912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,64,128,1,float16,float16,0,4.469002723693848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,64,0,1,float16,float16,0,12.505541483561197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,64,128,1,float16,fp8,0,4.440645217895508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,64,128,1,fp8,fp8,0,5.647178649902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,64,0,1,fp8,fp8,0,11.501370747884115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,64,128,1,float16,float16,0,2.471439997355143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,64,0,1,float16,fp8,0,12.518240610758463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,64,128,1,float16,fp8,0,2.4177120526631675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,64,128,1,fp8,fp8,0,3.1518398920694985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,64,128,1,float16,float16,0,2.0223679542541504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,64,0,1,float16,float16,0,6.577840169270833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,64,0,1,float16,float16,0,12.738479614257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,64,128,1,float16,fp8,0,1.989535967508952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,64,0,1,fp8,fp8,0,6.098512013753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,64,0,1,float16,fp8,0,6.70350964864095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,64,128,1,fp8,fp8,0,2.6617813110351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,64,0,1,fp8,fp8,0,11.93392562866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,64,0,1,float16,fp8,0,12.556500752766928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,64,128,1,float16,float16,0,2.071573257446289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,64,128,1,float16,fp8,0,2.042154630025228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,64,0,1,float16,float16,0,7.129568099975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,64,128,1,fp8,fp8,0,2.7188905080159507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,64,0,1,float16,fp8,0,6.700602849324544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,64,128,1,float16,float16,0,2.070074717203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,64,0,1,fp8,fp8,0,5.597328186035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,64,128,1,float16,fp8,0,2.030288060506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,64,0,1,float16,float16,0,6.550778706868489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,64,0,1,float16,fp8,0,6.370821634928386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,64,128,1,fp8,fp8,0,2.7382987340291343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,64,0,1,fp8,fp8,0,5.85977045694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,64,128,1,float16,float16,0,2.0810559590657554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,64,128,1,float16,fp8,0,2.0796000162760415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,64,0,1,float16,float16,0,6.441658655802409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,64,0,1,fp8,fp8,0,5.6731306711832685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,64,128,1,fp8,fp8,0,2.7520694732666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,64,0,1,float16,fp8,0,6.176912307739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,64,128,1,float16,float16,0,1.2176160017649333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,64,128,1,float16,fp8,0,1.1889333724975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,64,128,1,fp8,fp8,0,1.520261287689209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,64,0,1,float16,float16,0,6.57813835144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,64,0,1,float16,float16,0,3.398207982381185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,64,128,1,float16,float16,0,1.05404265721639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,64,0,1,fp8,fp8,0,5.686954498291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,64,0,1,fp8,fp8,0,3.006629308064779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,64,0,1,float16,fp8,0,6.5405012766520185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,64,128,1,float16,fp8,0,1.0515039761861165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,64,0,1,float16,fp8,0,3.402031898498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,64,128,1,fp8,fp8,0,1.4201226234436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,64,0,1,float16,float16,0,3.2310187021891275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,64,128,1,float16,float16,0,1.050335963567098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,64,128,1,float16,fp8,0,1.0524373054504395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,64,0,1,float16,fp8,0,3.324730555216471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,64,0,1,fp8,fp8,0,2.8337761561075845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,64,128,1,fp8,fp8,0,1.4077226320902507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,64,128,1,float16,float16,0,1.0547146797180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,64,0,1,float16,float16,0,3.1549545923868814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,64,128,1,float16,fp8,0,1.0907946427663167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,64,128,1,fp8,fp8,0,1.3744640350341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,64,0,1,fp8,fp8,0,2.8949705759684243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,64,0,1,float16,fp8,0,3.2513014475504556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,64,128,1,float16,float16,0,1.083301305770874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,64,0,1,float16,float16,0,3.24617067972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,64,128,1,float16,fp8,0,1.0472426414489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,64,0,1,float16,fp8,0,3.0809812545776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,64,0,1,fp8,fp8,0,2.8745813369750977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,64,128,1,float16,float16,0,0.6497226556142172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,64,128,1,fp8,fp8,0,1.3972907066345215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,64,128,1,float16,fp8,0,0.6392586628595988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,64,0,1,float16,float16,0,3.1865866978963218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,64,128,1,fp8,fp8,0,0.8201493422190348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,64,0,1,float16,float16,0,1.6626933415730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,64,128,1,float16,float16,0,0.5805066823959351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,64,128,1,float16,fp8,0,0.5750826597213745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,64,0,1,float16,fp8,0,1.6717759768168132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,64,0,1,fp8,fp8,0,2.8743467330932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,64,0,1,fp8,fp8,0,1.564186731974284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,64,0,1,float16,fp8,0,3.0902719497680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,64,0,1,float16,float16,0,1.6178347269694011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,64,128,1,fp8,fp8,0,0.7468480269114176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,64,128,1,float16,float16,0,0.5893333355585734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,64,128,1,fp8,fp8,0,0.756767988204956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,64,0,1,float16,fp8,0,1.6009972890218098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,64,128,1,float16,fp8,0,0.5870293378829956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,64,0,1,fp8,fp8,0,1.4755360285441081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,64,0,1,float16,float16,0,1.6332906087239583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,64,128,1,float16,float16,0,0.5877279837926229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,64,128,1,float16,fp8,0,0.5903040170669556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,64,128,1,fp8,fp8,0,0.7673066457112631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,64,0,1,float16,fp8,0,1.60152006149292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,64,0,1,fp8,fp8,0,1.4765760103861492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,64,128,1,float16,float16,0,0.5903520186742147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,64,0,1,float16,float16,0,1.6607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,64,128,1,float16,fp8,0,0.5865813493728638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,64,0,1,float16,fp8,0,1.5842132568359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,64,0,1,fp8,fp8,0,1.5216959317525227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,64,128,1,fp8,fp8,0,0.7607519626617432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,64,0,1,float16,float16,0,1.6395413080851238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,64,128,1,float16,float16,0,0.42377599080403644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,64,0,1,float16,fp8,0,1.6099467277526855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,64,128,1,float16,fp8,0,0.4185813268025716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,64,0,1,fp8,fp8,0,1.5052746136983235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,64,0,1,float16,float16,0,0.9429386456807455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,64,128,1,fp8,fp8,0,0.5070399840672811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,64,128,1,float16,float16,0,0.4171359936396281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,64,0,1,float16,fp8,0,0.9432960351308187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,64,0,1,fp8,fp8,0,0.8627200126647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,64,128,1,float16,fp8,0,0.41787199179331463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,64,128,1,fp8,fp8,0,0.5061706701914469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,64,0,1,float16,float16,0,0.9436799685160319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,64,128,1,float16,float16,0,0.4182080030441284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,64,128,1,float16,fp8,0,0.41918933391571045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,64,0,1,fp8,fp8,0,0.8486666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,64,0,1,float16,fp8,0,0.9568213621775309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,64,128,1,fp8,fp8,0,0.5058666865030924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,64,0,1,float16,float16,0,0.9430239995320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,64,128,1,float16,float16,0,0.4185653527577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,64,0,1,float16,fp8,0,0.9386186599731445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,64,0,1,fp8,fp8,0,0.8543093204498291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,64,128,1,float16,fp8,0,0.42081598440806073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,64,128,1,fp8,fp8,0,0.5053226550420126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,64,0,1,float16,float16,0,0.9386293093363444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,64,128,1,float16,float16,0,0.41650132338205975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,64,128,1,float16,fp8,0,0.42084264755249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,64,0,1,fp8,fp8,0,0.854202667872111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,64,0,1,float16,fp8,0,0.9445813496907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,64,128,1,fp8,fp8,0,0.5068159898122152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,64,0,1,float16,float16,0,0.9552106857299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,64,0,1,float16,fp8,0,0.9413173198699951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,64,0,1,fp8,fp8,0,0.8499306837717692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,64,128,1,float16,float16,0,5.662229537963867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,64,128,1,float16,fp8,0,5.621695836385091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,64,128,1,float16,float16,0,5.849514643351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,64,128,1,fp8,fp8,0,7.136810938517253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,64,0,1,fp8,fp8,0,11.209701538085938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,64,0,1,float16,float16,0,12.139882405598959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,64,128,1,float16,fp8,0,5.855893452962239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,64,0,1,float16,fp8,0,12.127562204996744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,64,128,1,fp8,fp8,0,7.4247252146403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,64,0,1,float16,float16,0,12.38040542602539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,64,128,1,float16,float16,0,5.844778696695964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,64,128,1,float16,fp8,0,5.828101476033528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,64,0,1,fp8,fp8,0,11.431621551513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,64,0,1,float16,fp8,0,12.409103393554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,64,128,1,fp8,fp8,0,7.41488520304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,64,128,1,float16,float16,0,5.895994822184245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,64,0,1,float16,float16,0,12.403556823730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,64,128,1,float16,fp8,0,5.881551742553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,64,0,1,float16,fp8,0,12.499231974283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,64,128,1,fp8,fp8,0,7.425418853759766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,64,128,1,float16,float16,0,3.3597707748413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,64,0,1,fp8,fp8,0,11.506224314371744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,64,128,1,float16,fp8,0,3.256789207458496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,64,0,1,float16,float16,0,12.525946299235025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,64,128,1,fp8,fp8,0,4.130549430847168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,64,0,1,float16,float16,0,6.71613883972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,64,0,1,float16,fp8,0,12.94867197672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,64,128,1,float16,float16,0,2.700415929158529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,64,128,1,float16,fp8,0,2.6293333371480307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,64,0,1,fp8,fp8,0,11.541461944580078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,64,0,1,fp8,fp8,0,6.558656056722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,64,128,1,fp8,fp8,0,3.5326080322265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,64,0,1,float16,fp8,0,7.154591878255208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,64,0,1,float16,float16,0,6.050879796346028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,64,128,1,float16,float16,0,2.8458986282348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,64,128,1,float16,fp8,0,2.8668107986450195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,64,0,1,float16,fp8,0,6.111461639404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,64,0,1,fp8,fp8,0,5.561808268229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,64,128,1,fp8,fp8,0,3.697882652282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,64,128,1,float16,float16,0,2.8566614786783853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,64,0,1,float16,float16,0,6.634597142537435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,64,128,1,float16,fp8,0,2.8683465321858725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,64,0,1,float16,fp8,0,6.184234619140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,64,128,1,fp8,fp8,0,3.73529052734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,64,0,1,fp8,fp8,0,5.8417174021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,64,0,1,float16,float16,0,6.380954742431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,64,128,1,float16,float16,0,2.9202400843302407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,64,128,1,float16,fp8,0,2.8761278788248696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,64,0,1,float16,fp8,0,6.272437413533528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,64,0,1,fp8,fp8,0,5.736085255940755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,64,128,1,fp8,fp8,0,3.762645403544108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,64,128,1,float16,float16,0,1.6194400787353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,64,128,1,float16,fp8,0,1.579653263092041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,64,0,1,float16,float16,0,6.514607747395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,64,128,1,fp8,fp8,0,2.0238986015319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,64,0,1,float16,float16,0,3.2492427825927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,64,128,1,float16,float16,0,1.3575572967529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,64,0,1,float16,fp8,0,6.824495951334636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,64,0,1,float16,fp8,0,3.449162801106771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,64,128,1,float16,fp8,0,1.325541337331136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,64,0,1,fp8,fp8,0,3.0451574325561523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,64,128,1,fp8,fp8,0,1.7740747133890789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,64,0,1,fp8,fp8,0,5.774074554443359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,64,0,1,float16,float16,0,3.247920036315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,64,128,1,float16,float16,0,1.4080479939778645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,64,128,1,float16,fp8,0,1.3361493746439617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,64,0,1,float16,fp8,0,3.0043465296427407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,64,0,1,fp8,fp8,0,2.7850774129231772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,64,128,1,fp8,fp8,0,1.7782185872395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,64,128,1,float16,float16,0,1.3768587112426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,64,0,1,float16,float16,0,3.012511889139811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,64,128,1,float16,fp8,0,1.3522826830546062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,64,0,1,float16,fp8,0,3.0832481384277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,64,0,1,fp8,fp8,0,2.7849706013997397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,64,128,1,fp8,fp8,0,1.8272693951924641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,64,0,1,float16,float16,0,3.1384960810343423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,64,128,1,float16,float16,0,1.3847306569417317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,64,128,1,float16,fp8,0,1.3571252822875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,64,0,1,float16,fp8,0,3.165914535522461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,64,0,1,fp8,fp8,0,2.771082560221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,64,128,1,fp8,fp8,0,1.842192014058431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,64,128,1,float16,float16,0,0.8142080307006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,64,128,1,float16,fp8,0,0.8029706478118896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,64,0,1,float16,float16,0,3.027573267618815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,64,128,1,fp8,fp8,0,1.0312799612681072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,64,0,1,float16,float16,0,1.6970507303873699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,64,0,1,float16,fp8,0,3.0773280461629233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,64,128,1,float16,float16,0,0.7217706839243571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,64,0,1,fp8,fp8,0,2.8074239095052085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,64,0,1,float16,fp8,0,1.6503146489461262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,64,0,1,fp8,fp8,0,1.5542880694071453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,64,128,1,float16,fp8,0,0.7253119945526123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,64,0,1,float16,float16,0,1.5453119277954102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,64,128,1,fp8,fp8,0,0.9719573656717936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,64,128,1,float16,float16,0,0.7329866886138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,64,128,1,float16,fp8,0,0.7203733126322428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,64,0,1,float16,fp8,0,1.5320213635762532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,64,0,1,fp8,fp8,0,1.4169227282206218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,64,0,1,float16,float16,0,1.5686453183492024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,64,128,1,fp8,fp8,0,0.9541760285695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,64,128,1,float16,float16,0,0.7347679932912191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,64,0,1,float16,fp8,0,1.5655840237935383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,64,0,1,fp8,fp8,0,1.44268798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,64,128,1,float16,fp8,0,0.7024906476338705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,64,0,1,float16,float16,0,1.5760213534037273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,64,128,1,fp8,fp8,0,0.9395733674367269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,64,128,1,float16,float16,0,0.7333599726359049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,64,128,1,float16,fp8,0,0.7191147009531657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,64,0,1,float16,fp8,0,1.5401760737101238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,64,0,1,fp8,fp8,0,1.4590773582458496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,64,128,1,fp8,fp8,0,0.9440320332845052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,64,128,1,float16,float16,0,0.43937599658966064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,64,0,1,float16,float16,0,1.5533119837443035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,64,128,1,float16,fp8,0,0.43778133392333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,64,128,1,fp8,fp8,0,0.5639786720275879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,64,0,1,float16,fp8,0,1.5705173810323079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,64,0,1,float16,float16,0,0.852895975112915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,64,0,1,fp8,fp8,0,1.453887939453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,64,128,1,float16,float16,0,0.4020373423894246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,64,0,1,float16,fp8,0,0.855077346165975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,64,0,1,fp8,fp8,0,0.8164373238881429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,64,128,1,float16,fp8,0,0.39443198839823407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,64,128,1,fp8,fp8,0,0.5228053331375122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,64,0,1,float16,float16,0,0.8214293320973715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,64,0,1,float16,fp8,0,0.813701311747233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,64,128,1,float16,float16,0,0.39813868204752606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,64,0,1,fp8,fp8,0,0.7672586441040039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,64,128,1,float16,fp8,0,0.39526931444803876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,64,128,1,fp8,fp8,0,0.5176213184992472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,64,0,1,float16,float16,0,0.8197706540425619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,64,0,1,float16,fp8,0,0.8157440026601156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,64,128,1,float16,float16,0,0.39949333667755127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,64,0,1,fp8,fp8,0,0.7682399749755859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,64,128,1,float16,fp8,0,0.3941333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,64,0,1,float16,float16,0,0.8203519980112711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,64,128,1,fp8,fp8,0,0.518175999323527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,64,128,1,float16,float16,0,0.40275200208028156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,64,0,1,float16,fp8,0,0.8173387050628662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,64,0,1,fp8,fp8,0,0.7691360314687093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,64,128,1,float16,fp8,0,0.39778133233388263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,64,0,1,float16,float16,0,0.8223840395609537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,64,128,1,fp8,fp8,0,0.516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,64,128,1,float16,float16,0,0.2887679934501648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,64,0,1,float16,fp8,0,0.8204747041066488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,64,0,1,fp8,fp8,0,0.7740800380706787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,64,0,1,float16,float16,0,0.5051519870758057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,64,128,1,float16,fp8,0,0.2883146603902181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,64,128,1,fp8,fp8,0,0.3510773181915283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,64,0,1,float16,fp8,0,0.5048533280690511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,64,0,1,fp8,fp8,0,0.45881601174672443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,64,128,1,float16,float16,0,0.2873973250389099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,64,0,1,float16,float16,0,0.5067840019861857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,64,128,1,float16,fp8,0,0.2868959903717041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,64,128,1,fp8,fp8,0,0.3493066628774007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,64,0,1,float16,fp8,0,0.5053280194600424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,64,0,1,fp8,fp8,0,0.4574879805246989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,64,128,1,float16,float16,0,0.28650667270024616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,64,0,1,float16,float16,0,0.5057973464330038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,64,128,1,float16,fp8,0,0.28760000069936115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,64,128,1,fp8,fp8,0,0.34980801741282147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,64,0,1,float16,fp8,0,0.5089493195215861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,64,0,1,fp8,fp8,0,0.4591093460718791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,64,128,1,float16,float16,0,0.2868906656901042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,64,128,1,float16,fp8,0,0.2873386740684509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,64,0,1,float16,float16,0,0.506218671798706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,64,128,1,fp8,fp8,0,0.34914668401082355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,64,0,1,float16,fp8,0,0.5074133475621542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,64,0,1,fp8,fp8,0,0.4579840103785197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,64,128,1,float16,float16,0,0.2880586584409078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,64,0,1,float16,float16,0,0.5075680017471313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,64,128,1,float16,fp8,0,0.28782399495442706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,64,128,1,fp8,fp8,0,0.3508106470108032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,64,0,1,float16,fp8,0,0.5073279937108358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,64,0,1,fp8,fp8,0,0.45841066042582196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,64,128,1,float16,float16,0,4.125631968180339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,64,128,1,float16,fp8,0,4.08896541595459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,64,128,1,float16,float16,0,4.236490567525228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,64,128,1,fp8,fp8,0,5.273887952168782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,64,0,1,fp8,fp8,0,6.863397598266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,64,0,1,float16,float16,0,7.497664133707683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,64,0,1,float16,fp8,0,7.419776280721028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,64,128,1,float16,fp8,0,4.2078507741292315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,64,128,1,fp8,fp8,0,5.373301188151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,64,0,1,float16,float16,0,7.6769224802653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,64,128,1,float16,float16,0,4.256229400634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,64,128,1,float16,fp8,0,4.239306767781575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,64,0,1,float16,fp8,0,7.555013020833333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,64,128,1,fp8,fp8,0,5.361610412597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,64,0,1,fp8,fp8,0,6.982522964477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,64,0,1,float16,float16,0,7.694821039835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,64,128,1,float16,float16,0,4.2835893630981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,64,128,1,float16,fp8,0,4.271653175354004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,64,0,1,float16,fp8,0,7.671215693155925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,64,0,1,fp8,fp8,0,7.067429224650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,64,128,1,float16,float16,0,2.4570239384969077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,64,128,1,fp8,fp8,0,5.489936192830403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,64,0,1,float16,float16,0,7.764890670776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,64,128,1,float16,fp8,0,2.4023946126302085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,64,0,1,float16,float16,0,4.27238400777181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,64,128,1,fp8,fp8,0,3.091893196105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,64,128,1,float16,float16,0,1.9965813954671223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,64,0,1,fp8,fp8,0,7.143290837605794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,64,0,1,float16,fp8,0,7.609104156494141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,64,0,1,float16,fp8,0,4.2359466552734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,64,0,1,fp8,fp8,0,3.872111956278483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,64,128,1,float16,fp8,0,2.0093493461608887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,64,128,1,fp8,fp8,0,2.58243195215861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,64,0,1,float16,float16,0,3.7060426076253257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,64,128,1,float16,float16,0,2.0067572593688965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,64,0,1,fp8,fp8,0,3.3840694427490234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,64,0,1,float16,fp8,0,3.753690719604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,64,128,1,float16,fp8,0,1.9832693735758464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,64,128,1,fp8,fp8,0,2.6799999872843423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,64,128,1,float16,float16,0,2.0153013865152993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,64,0,1,float16,float16,0,3.712010701497396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,64,128,1,float16,fp8,0,2.0155946413675943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,64,0,1,fp8,fp8,0,3.4419787724812827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,64,0,1,float16,fp8,0,3.8583733240763345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,64,128,1,fp8,fp8,0,2.6475839614868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,64,128,1,float16,float16,0,2.074618657430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,64,0,1,float16,float16,0,4.61301326751709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,64,0,1,float16,fp8,0,3.748586654663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,64,128,1,float16,fp8,0,2.024517377217611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,64,0,1,fp8,fp8,0,3.4390026728312173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,64,128,1,float16,float16,0,1.263434648513794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,64,128,1,fp8,fp8,0,2.7596108118693032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,64,128,1,float16,fp8,0,1.1731680234273274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,64,0,1,float16,float16,0,3.856517473856608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,64,128,1,fp8,fp8,0,1.5166346232096355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,64,0,1,float16,fp8,0,3.713839848836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,64,0,1,float16,float16,0,2.099674701690674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,64,0,1,fp8,fp8,0,3.5154666900634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,64,128,1,float16,float16,0,1.0153813362121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,64,0,1,float16,fp8,0,2.085338592529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,64,0,1,fp8,fp8,0,1.900437355041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,64,128,1,float16,fp8,0,1.0272640387217205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,64,0,1,float16,float16,0,1.8654239972432454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,64,128,1,fp8,fp8,0,1.3353439966837566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,64,128,1,float16,float16,0,1.0192320346832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,64,128,1,float16,fp8,0,1.0131093660990398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,64,0,1,fp8,fp8,0,1.7282239596048992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,64,0,1,float16,fp8,0,1.852549393971761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,64,128,1,fp8,fp8,0,1.3403840065002441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,64,0,1,float16,float16,0,1.8801226615905762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,64,128,1,float16,float16,0,1.0255413055419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,64,128,1,float16,fp8,0,1.0157653490702312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,64,0,1,fp8,fp8,0,1.7361173629760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,64,0,1,float16,fp8,0,1.8533066113789876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,64,0,1,float16,float16,0,1.873429298400879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,64,128,1,fp8,fp8,0,1.350096066792806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,64,0,1,float16,fp8,0,1.8594932556152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,64,128,1,float16,float16,0,1.0382400353749592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,64,128,1,float16,fp8,0,1.0255253314971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,64,0,1,fp8,fp8,0,1.7462399800618489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,64,128,1,fp8,fp8,0,1.3459146817525227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,64,0,1,float16,float16,0,1.8805386225382488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,64,128,1,float16,float16,0,0.6208906571070353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,64,0,1,float16,fp8,0,1.865455945332845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,64,0,1,float16,float16,0,1.0428586800893147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,64,128,1,float16,fp8,0,0.6096373399098715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,64,0,1,fp8,fp8,0,1.750864028930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,64,128,1,fp8,fp8,0,0.7756906350453695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,64,128,1,float16,float16,0,0.5416266520818075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,64,0,1,float16,fp8,0,1.0327786604563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,64,0,1,fp8,fp8,0,0.974565347035726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,64,0,1,float16,fp8,0,0.9612267017364502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,64,128,1,float16,fp8,0,0.540336012840271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,64,0,1,float16,float16,0,0.9668746789296468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,64,128,1,fp8,fp8,0,0.7039999961853027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,64,128,1,float16,float16,0,0.5464906692504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,64,0,1,fp8,fp8,0,0.9030933380126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,64,128,1,float16,fp8,0,0.5360159873962402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,64,0,1,float16,float16,0,0.9691466490427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,64,128,1,fp8,fp8,0,0.7138559818267822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,64,128,1,float16,float16,0,0.5480373303095499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,64,0,1,float16,fp8,0,0.9618293444315592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,64,0,1,fp8,fp8,0,0.905898650487264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,64,128,1,float16,fp8,0,0.5388799905776978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,64,0,1,float16,float16,0,0.9737280209859213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,64,128,1,fp8,fp8,0,0.7065813541412354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,64,0,1,float16,fp8,0,0.978111982345581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,64,128,1,float16,float16,0,0.5487200021743774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,64,0,1,fp8,fp8,0,0.9092746575673422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,64,128,1,float16,fp8,0,0.5424960056940714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,64,0,1,float16,float16,0,0.9762880007425944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,64,128,1,fp8,fp8,0,0.7124053637186686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,64,0,1,float16,fp8,0,0.967082659403483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,64,128,1,float16,float16,0,0.3330186605453491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,64,0,1,fp8,fp8,0,0.9152320226033529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,64,0,1,float16,float16,0,0.549018661181132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,64,128,1,float16,fp8,0,0.333349347114563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,64,128,1,fp8,fp8,0,0.4267306725184123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,64,0,1,float16,fp8,0,0.5493813355763754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,64,0,1,fp8,fp8,0,0.5267359813054403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,64,128,1,float16,float16,0,0.30435200532277423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,64,0,1,float16,float16,0,0.5238240162531534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,64,128,1,float16,fp8,0,0.2998453378677368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,64,128,1,fp8,fp8,0,0.4012639919916789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,64,0,1,float16,fp8,0,0.5193386475245158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,64,0,1,fp8,fp8,0,0.475872000058492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,64,128,1,float16,float16,0,0.3092586596806844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,64,128,1,float16,fp8,0,0.30211732784907025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,64,0,1,float16,float16,0,0.5223413308461508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,64,128,1,fp8,fp8,0,0.39344000816345215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,64,0,1,float16,fp8,0,0.5265173514684042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,64,128,1,float16,float16,0,0.30643200874328613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,64,0,1,fp8,fp8,0,0.475109338760376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,64,0,1,float16,float16,0,0.5258293151855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,64,128,1,float16,fp8,0,0.3062826593716939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,64,128,1,fp8,fp8,0,0.3965013424555461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,64,0,1,float16,fp8,0,0.5216373205184937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,64,128,1,float16,float16,0,0.3115466634432475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,64,0,1,fp8,fp8,0,0.4788000186284383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,64,0,1,float16,float16,0,0.5275253454844157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,64,0,1,float16,fp8,0,0.5292693376541138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,64,128,1,float16,fp8,0,0.30434133609135944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,64,128,1,fp8,fp8,0,0.3994506597518921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,64,128,1,float16,float16,0,0.22251200675964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,64,0,1,fp8,fp8,0,0.481386661529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,64,128,1,float16,fp8,0,0.22299200296401978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,64,0,1,float16,float16,0,0.32918934027353924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,64,128,1,fp8,fp8,0,0.27298132578531903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,64,0,1,float16,fp8,0,0.3280533353487651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,64,0,1,fp8,fp8,0,0.3048746585845947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,64,128,1,float16,float16,0,0.22138667106628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,64,128,1,float16,float16,0,0.22182933489481607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,64,0,1,float16,float16,0,0.31851200262705487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,64,128,1,float16,fp8,0,0.2214133342107137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,64,128,1,fp8,fp8,0,0.2715946634610494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,64,0,1,float16,fp8,0,0.3205653429031372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,64,0,1,fp8,fp8,0,0.3032426635424296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,64,0,1,float16,float16,0,0.31863999366760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,64,0,1,float16,float16,0,0.32022400697072345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,64,128,1,float16,fp8,0,0.22148799896240234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,64,128,1,fp8,fp8,0,0.2707413236300151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,64,0,1,float16,fp8,0,0.3184853394826253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,64,0,1,fp8,fp8,0,0.3035893241564433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,64,128,1,float16,float16,0,0.2217866579691569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,64,128,1,float16,fp8,0,0.22058133284250894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,64,128,1,fp8,fp8,0,0.27133333683013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,64,0,1,float16,fp8,0,0.31969066460927326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,64,0,1,fp8,fp8,0,0.3048373262087504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,64,128,1,float16,float16,0,0.22163732846577963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,64,0,1,float16,float16,0,0.3190133372942607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,64,128,1,float16,fp8,0,0.2219946583112081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,64,128,1,fp8,fp8,0,0.2711626688639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,64,0,1,float16,fp8,0,0.3192266623179118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,64,0,1,fp8,fp8,0,0.30398400624593097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,64,128,1,float16,float16,0,5.5612748463948565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,64,128,1,float16,fp8,0,5.431626637776692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,64,128,1,float16,float16,0,5.528645197550456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,64,128,1,fp8,fp8,0,6.815637588500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,64,0,1,float16,float16,0,7.963722864786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,64,0,1,fp8,fp8,0,7.126282374064128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,64,0,1,float16,fp8,0,7.8619944254557295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,64,0,1,float16,float16,0,7.987744013468425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,64,128,1,float16,fp8,0,5.45469347635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,64,128,1,fp8,fp8,0,6.919562657674153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,64,128,1,float16,float16,0,5.540746688842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,64,128,1,float16,fp8,0,5.452725092569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,64,0,1,fp8,fp8,0,7.270997365315755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,64,0,1,float16,fp8,0,7.901125590006511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,64,0,1,float16,float16,0,8.058021545410156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,64,128,1,fp8,fp8,0,7.081439971923828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,64,0,1,float16,fp8,0,7.932773590087891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,64,128,1,float16,float16,0,5.55677858988444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,64,128,1,float16,fp8,0,5.529120127360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,64,0,1,fp8,fp8,0,7.332405090332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,64,128,1,fp8,fp8,0,7.158901214599609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,64,0,1,float16,float16,0,8.129472096761068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,64,128,1,float16,float16,0,3.246570587158203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,64,128,1,float16,fp8,0,3.1279147466023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,64,0,1,fp8,fp8,0,7.43008550008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,64,0,1,float16,fp8,0,7.981333414713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,64,0,1,float16,float16,0,4.463450749715169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,64,128,1,fp8,fp8,0,4.02732785542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,64,128,1,float16,float16,0,2.611658732096354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,64,0,1,float16,fp8,0,4.4464108149210615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,64,128,1,float16,fp8,0,2.599722703297933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,64,0,1,fp8,fp8,0,4.204997380574544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,64,0,1,float16,float16,0,4.016826629638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,64,128,1,fp8,fp8,0,3.4533920288085938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,64,128,1,float16,float16,0,2.711674690246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,64,0,1,float16,fp8,0,3.849151929219564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,64,128,1,float16,fp8,0,2.6918614705403647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,64,0,1,fp8,fp8,0,3.5306666692097983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,64,0,1,float16,float16,0,3.9394667943318686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,64,128,1,fp8,fp8,0,3.492016156514486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,64,0,1,fp8,fp8,0,3.675818761189779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,64,0,1,float16,fp8,0,3.934645334879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,64,128,1,float16,float16,0,2.7259734471639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,64,128,1,float16,fp8,0,2.706101417541504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,64,128,1,fp8,fp8,0,3.454981486002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,64,0,1,float16,float16,0,3.948879877726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,64,128,1,float16,float16,0,2.758629480997721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,64,0,1,fp8,fp8,0,3.6452693939208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,64,0,1,float16,fp8,0,4.089328130086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,64,128,1,float16,fp8,0,2.72923215230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,64,0,1,float16,float16,0,3.983503977457682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,64,128,1,fp8,fp8,0,3.5363521575927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,64,128,1,float16,float16,0,1.5989866256713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,64,128,1,float16,fp8,0,1.5632640520731609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,64,0,1,float16,float16,0,2.269477367401123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,64,128,1,fp8,fp8,0,1.9854826927185059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,64,0,1,fp8,fp8,0,3.670005480448405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,64,0,1,float16,fp8,0,3.9505386352539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,64,0,1,float16,fp8,0,2.1712800661722818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,64,128,1,float16,float16,0,1.3125706513722737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,64,0,1,fp8,fp8,0,2.0683679580688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,64,128,1,fp8,fp8,0,1.7055573463439941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,64,0,1,float16,float16,0,1.9180906613667805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,64,128,1,float16,fp8,0,1.3278079827626545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,64,0,1,float16,fp8,0,1.900762716929118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,64,0,1,fp8,fp8,0,1.7889493306477864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,64,128,1,float16,float16,0,1.3605012893676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,64,128,1,float16,fp8,0,1.3098986943562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,64,0,1,float16,float16,0,1.934384028116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,64,128,1,fp8,fp8,0,1.7087732950846355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,64,0,1,float16,fp8,0,1.9059146245320637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,64,0,1,fp8,fp8,0,1.8061866760253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,64,128,1,float16,float16,0,1.3265386422475178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,64,128,1,float16,fp8,0,1.327946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,64,0,1,float16,float16,0,1.9766027132670085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,64,128,1,fp8,fp8,0,1.732063929239909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,64,128,1,float16,float16,0,1.345695972442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,64,0,1,fp8,fp8,0,1.782709280649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,64,0,1,float16,fp8,0,1.9236373901367188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,64,128,1,float16,fp8,0,1.3199679851531982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,64,0,1,float16,float16,0,1.9596586227416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,64,128,1,float16,float16,0,0.8049600124359131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,64,128,1,fp8,fp8,0,1.7707093556722004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,64,0,1,float16,float16,0,1.120479981104533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,64,128,1,float16,fp8,0,0.7864533265431722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,64,0,1,float16,fp8,0,1.9439573287963867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,64,0,1,fp8,fp8,0,1.820042610168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,64,128,1,fp8,fp8,0,0.9848960240681967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,64,0,1,float16,fp8,0,1.0986186663309734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,64,0,1,fp8,fp8,0,1.0211199919382732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,64,128,1,float16,float16,0,0.6886826356252035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,64,0,1,float16,float16,0,1.0075733661651611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,64,128,1,float16,fp8,0,0.6696853637695312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,64,0,1,float16,fp8,0,0.9819626808166504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,64,128,1,fp8,fp8,0,0.8945546944936117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,64,128,1,float16,float16,0,0.6929013729095459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,64,0,1,fp8,fp8,0,0.9174453417460123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,64,128,1,float16,fp8,0,0.6812533537546793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,64,0,1,float16,float16,0,0.9966026941935221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,64,128,1,fp8,fp8,0,0.8991626898447672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,64,128,1,float16,float16,0,0.6919466654459635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,64,0,1,fp8,fp8,0,0.927722692489624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,64,0,1,float16,fp8,0,0.9876053333282471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,64,128,1,float16,fp8,0,0.6816373666127523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,64,0,1,float16,float16,0,1.0036479632059734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,64,128,1,fp8,fp8,0,0.9017013708750407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,64,0,1,float16,fp8,0,0.9887573719024658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,64,128,1,float16,float16,0,0.6930399735768636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,64,0,1,fp8,fp8,0,0.9336746533711752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,64,0,1,float16,fp8,0,0.9982453187306722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,64,128,1,float16,fp8,0,0.68941330909729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,64,0,1,float16,float16,0,0.9966933727264404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,64,128,1,fp8,fp8,0,0.9044373035430908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,64,128,1,float16,float16,0,0.42005332310994464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,64,0,1,fp8,fp8,0,0.9374933242797852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,64,128,1,float16,fp8,0,0.41436266899108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,64,0,1,float16,float16,0,0.5744533141454061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,64,128,1,fp8,fp8,0,0.5306506554285685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,64,0,1,float16,fp8,0,0.5700533390045166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,64,128,1,float16,float16,0,0.3673119942347209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,64,0,1,fp8,fp8,0,0.5364693403244019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,64,128,1,float16,fp8,0,0.36152533690134686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,64,0,1,float16,float16,0,0.5253173510233561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,64,128,1,fp8,fp8,0,0.480954647064209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,64,0,1,float16,fp8,0,0.5183039903640747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,64,0,1,fp8,fp8,0,0.4921333392461141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,64,128,1,float16,float16,0,0.3688960075378418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,64,128,1,float16,fp8,0,0.36246931552886963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,64,0,1,float16,float16,0,0.5259360074996948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,64,128,1,fp8,fp8,0,0.4787840048472087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,64,0,1,float16,fp8,0,0.521722674369812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,64,0,1,fp8,fp8,0,0.4920426607131958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,64,128,1,float16,float16,0,0.37191998958587646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,64,128,1,float16,fp8,0,0.36451733112335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,64,0,1,float16,float16,0,0.5268906752268473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,64,128,1,fp8,fp8,0,0.4817013343175252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,64,0,1,float16,fp8,0,0.5237760146458944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,64,0,1,fp8,fp8,0,0.49482134977976483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,64,128,1,float16,float16,0,0.3723413149515788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,64,0,1,float16,float16,0,0.5297013521194458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,64,128,1,float16,fp8,0,0.3684533437093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,64,128,1,fp8,fp8,0,0.4848586718241374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,64,0,1,float16,fp8,0,0.5249546766281128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,64,0,1,fp8,fp8,0,0.4967679977416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,64,128,1,float16,float16,0,0.2311840057373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,64,0,1,float16,float16,0,0.31009600559870404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,64,128,1,float16,fp8,0,0.2304746707280477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,64,128,1,fp8,fp8,0,0.2964640061060588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,64,0,1,float16,fp8,0,0.31113600730895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,64,0,1,fp8,fp8,0,0.2892799973487854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,64,128,1,float16,float16,0,0.2096959948539734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,64,0,1,float16,float16,0,0.2925493319829305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,64,128,1,float16,fp8,0,0.2083359956741333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,64,128,1,fp8,fp8,0,0.2730240027109782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,64,0,1,float16,fp8,0,0.29072533051172894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,64,0,1,fp8,fp8,0,0.2654026746749878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,64,128,1,float16,float16,0,0.21008533239364624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,64,0,1,float16,float16,0,0.29148799180984497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,64,128,1,float16,fp8,0,0.20894932746887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,64,128,1,fp8,fp8,0,0.2732693354288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,64,0,1,float16,float16,0,0.2927359938621521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,64,0,1,float16,fp8,0,0.2905386686325073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,64,0,1,fp8,fp8,0,0.2666400074958801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,64,128,1,float16,float16,0,0.21166932582855225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,64,128,1,float16,fp8,0,0.2090453306833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,64,128,1,fp8,fp8,0,0.2752266724904378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,64,0,1,float16,fp8,0,0.28999465703964233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,64,0,1,fp8,fp8,0,0.26659733057022095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,64,128,1,float16,float16,0,0.21291200319925943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,64,0,1,float16,float16,0,0.2949120004971822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,64,128,1,float16,fp8,0,0.2105706731478373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,64,128,1,fp8,fp8,0,0.2759733398755391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,64,0,1,float16,fp8,0,0.29356799523035687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,64,0,1,fp8,fp8,0,0.2688533266385396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,64,128,1,float16,float16,0,0.1574613352616628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,64,0,1,float16,float16,0,0.19261866807937622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,64,128,1,float16,fp8,0,0.15796266992886862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,64,128,1,fp8,fp8,0,0.1939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,64,0,1,float16,fp8,0,0.19242133696873984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,64,0,1,float16,fp8,0,0.1863093376159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,64,0,1,fp8,fp8,0,0.1804693341255188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,64,128,1,float16,float16,0,0.15438933173815408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,64,0,1,float16,float16,0,0.18570667505264282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,64,128,1,float16,fp8,0,0.1545866628487905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,64,128,1,fp8,fp8,0,0.1848106582959493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,64,0,1,fp8,fp8,0,0.17772799730300903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,64,128,1,float16,float16,0,0.1552906632423401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,64,0,1,float16,float16,0,0.186191995938619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,64,128,1,float16,fp8,0,0.15482133626937866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,64,128,1,fp8,fp8,0,0.19134400288263956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,64,0,1,float16,fp8,0,0.18568533658981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,64,0,1,fp8,fp8,0,0.17903467019399008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,64,128,1,float16,float16,0,0.1556426684061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,64,0,1,float16,float16,0,0.1863200068473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,64,128,1,float16,fp8,0,0.1553386648495992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,64,128,1,fp8,fp8,0,0.1906986633936564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,64,0,1,float16,fp8,0,0.18658133347829184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,64,0,1,fp8,fp8,0,0.17876267433166504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,64,128,1,float16,float16,0,0.1557973325252533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,64,0,1,float16,float16,0,0.18662933508555093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,64,128,1,float16,fp8,0,0.15574399630228677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,64,128,1,fp8,fp8,0,0.1902880072593689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,64,0,1,float16,fp8,0,0.18782933553059897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,64,0,1,fp8,fp8,0,0.17918399969736734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,64,128,1,float16,float16,0,4.08073075612386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,64,128,1,float16,fp8,0,4.019599914550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,64,128,1,float16,float16,0,4.03930123647054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,64,0,1,fp8,fp8,0,4.6653493245442705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,64,0,1,float16,float16,0,5.213717460632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,64,128,1,fp8,fp8,0,5.089152018229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,64,0,1,float16,fp8,0,5.1582291920979815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,64,0,1,float16,float16,0,5.160981178283691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,64,128,1,float16,fp8,0,3.965008099873861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,64,128,1,float16,float16,0,4.050869305928548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,64,128,1,fp8,fp8,0,5.1178239186604815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,64,128,1,float16,fp8,0,4.000293413798015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,64,0,1,fp8,fp8,0,4.670426686604817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,64,0,1,float16,fp8,0,5.119477272033691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,64,0,1,float16,float16,0,5.182959874471028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,64,128,1,fp8,fp8,0,5.111509323120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,64,0,1,float16,fp8,0,5.126693407694499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,64,128,1,float16,float16,0,4.064714749654134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,64,128,1,float16,fp8,0,4.006650606791179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,64,0,1,fp8,fp8,0,4.683690706888835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,64,0,1,float16,float16,0,5.351392110188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,64,128,1,fp8,fp8,0,5.124544143676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,64,0,1,float16,fp8,0,5.183594703674316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,64,128,1,float16,float16,0,2.3739466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,64,128,1,float16,fp8,0,2.3292320569356284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,64,0,1,float16,float16,0,2.955530802408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,64,0,1,fp8,fp8,0,4.696853319803874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,64,128,1,fp8,fp8,0,3.010437329610189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,64,128,1,float16,float16,0,1.923232078552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,64,0,1,float16,fp8,0,3.0250186920166016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,64,0,1,fp8,fp8,0,2.7639732360839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,64,128,1,float16,fp8,0,1.912864049275716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,64,0,1,float16,float16,0,2.492304007212321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,64,128,1,fp8,fp8,0,2.499370733896891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,64,0,1,fp8,fp8,0,2.2798825899759927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,64,0,1,float16,fp8,0,2.4892800649007163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,64,128,1,float16,float16,0,2.0151146252950034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,64,128,1,float16,fp8,0,1.9580480257670085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,64,0,1,float16,float16,0,2.5487093925476074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,64,128,1,fp8,fp8,0,2.5527626673380532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,64,0,1,float16,fp8,0,2.5243679682413735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,64,128,1,float16,float16,0,1.9644907315572102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,64,0,1,fp8,fp8,0,2.3099519411722818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,64,128,1,float16,fp8,0,1.9733866055806477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,64,0,1,float16,float16,0,2.553599993387858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,64,128,1,fp8,fp8,0,2.554650624593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,64,128,1,float16,float16,0,1.9953440030415852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,64,0,1,fp8,fp8,0,2.412399927775065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,64,0,1,float16,fp8,0,2.539781411488851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,64,128,1,float16,fp8,0,1.9599199295043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,64,0,1,float16,float16,0,2.5719146728515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,64,128,1,float16,float16,0,1.184597333272298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,64,128,1,fp8,fp8,0,2.6023732821146646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,64,128,1,float16,fp8,0,1.1585813363393147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,64,0,1,float16,fp8,0,2.5490026473999023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,64,0,1,float16,float16,0,1.5076799392700195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,64,0,1,fp8,fp8,0,2.3550987243652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,64,128,1,fp8,fp8,0,1.4411892890930176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,64,0,1,float16,fp8,0,1.4437546730041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,64,128,1,float16,float16,0,0.9916853109995524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,64,128,1,float16,fp8,0,0.9750773111979166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,64,0,1,fp8,fp8,0,1.3327893416086833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,64,0,1,float16,float16,0,1.2730080286661785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,64,128,1,fp8,fp8,0,1.3249173164367676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,64,0,1,float16,fp8,0,1.2491146723429363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,64,0,1,fp8,fp8,0,1.157040039698283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,64,128,1,float16,float16,0,0.9811519781748453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,64,128,1,float16,fp8,0,0.9796799818674723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,64,0,1,float16,float16,0,1.2766613165537517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,64,128,1,fp8,fp8,0,1.2838079929351807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,64,0,1,float16,fp8,0,1.2613120079040527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,64,128,1,float16,float16,0,0.9909546375274658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,64,0,1,fp8,fp8,0,1.1790826320648193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,64,0,1,float16,float16,0,1.2669920126597087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,64,128,1,float16,fp8,0,0.9725653330485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,64,128,1,fp8,fp8,0,1.2967039744059246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,64,0,1,float16,fp8,0,1.2585279941558838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,64,0,1,fp8,fp8,0,1.1732640266418457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,64,128,1,float16,float16,0,1.004325310389201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,64,128,1,float16,fp8,0,0.9940213362375895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,64,0,1,float16,float16,0,1.2793760299682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,64,128,1,fp8,fp8,0,1.2897813320159912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,64,128,1,float16,float16,0,0.6022719939549764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,64,0,1,float16,fp8,0,1.2603413263956706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,64,0,1,float16,float16,0,0.745247999827067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,64,128,1,float16,fp8,0,0.595631996790568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,64,0,1,fp8,fp8,0,1.1895946661631267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,64,128,1,fp8,fp8,0,0.7471520105997721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,64,0,1,float16,fp8,0,0.7328426837921143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,64,128,1,float16,float16,0,0.5096586545308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,64,0,1,fp8,fp8,0,0.6767733097076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,64,0,1,float16,float16,0,0.6526506741841634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,64,128,1,float16,fp8,0,0.505232016245524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,64,0,1,float16,fp8,0,0.6516426801681519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,64,128,1,fp8,fp8,0,0.671989361445109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,64,128,1,float16,float16,0,0.5147733290990194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,64,0,1,fp8,fp8,0,0.6099786758422852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,64,128,1,float16,fp8,0,0.5052533149719238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,64,0,1,float16,float16,0,0.6549599965413412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,64,128,1,fp8,fp8,0,0.6653386751810709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,64,0,1,float16,fp8,0,0.6499840021133423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,64,0,1,fp8,fp8,0,0.612122654914856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,64,128,1,float16,float16,0,0.5209546486536661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,64,0,1,float16,float16,0,0.6597386598587036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,64,128,1,float16,fp8,0,0.5134559869766235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,64,128,1,fp8,fp8,0,0.6691413720448812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,64,0,1,float16,fp8,0,0.6494666735331217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,64,0,1,fp8,fp8,0,0.6074026823043823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,64,128,1,float16,float16,0,0.5216106573740641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,64,0,1,float16,float16,0,0.6681013107299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,64,128,1,float16,fp8,0,0.5175573428471884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,64,128,1,fp8,fp8,0,0.6761813163757324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,64,128,1,float16,float16,0,0.31758399804433185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,64,0,1,float16,fp8,0,0.6617973248163859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,64,0,1,fp8,fp8,0,0.6112746795018514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,64,128,1,float16,fp8,0,0.31242666641871136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,64,0,1,float16,float16,0,0.3930559953053792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,64,128,1,fp8,fp8,0,0.4026133219401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,64,0,1,float16,fp8,0,0.3872426748275757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,64,0,1,fp8,fp8,0,0.3656853437423706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,64,128,1,float16,float16,0,0.27689067522684735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,64,0,1,float16,float16,0,0.3500800132751465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,64,128,1,float16,fp8,0,0.27552000681559247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,64,128,1,fp8,fp8,0,0.36405332883199054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,64,0,1,float16,fp8,0,0.3516800006230672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,64,0,1,fp8,fp8,0,0.3209279974301656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,64,128,1,float16,float16,0,0.2845013340314229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,64,0,1,float16,float16,0,0.3538666566212972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,64,128,1,float16,fp8,0,0.27694932619730633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,64,128,1,float16,float16,0,0.28168533245722455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,64,128,1,fp8,fp8,0,0.3667999903361003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,64,0,1,float16,fp8,0,0.3506186803181966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,64,0,1,fp8,fp8,0,0.3222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,64,0,1,float16,float16,0,0.3595893383026123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,64,128,1,float16,fp8,0,0.27847999334335327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,64,128,1,fp8,fp8,0,0.3668373425801595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,64,0,1,float16,fp8,0,0.3519306580225627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,64,0,1,fp8,fp8,0,0.3247680068016052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,64,128,1,float16,float16,0,0.2837439974149068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,64,0,1,float16,float16,0,0.36398398876190186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,64,128,1,float16,fp8,0,0.28362133105595905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,64,128,1,fp8,fp8,0,0.3707733154296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,64,0,1,float16,fp8,0,0.35339200496673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,64,0,1,fp8,fp8,0,0.3277706702550252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,64,128,1,float16,float16,0,0.17804267009099325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,64,0,1,float16,float16,0,0.22666666905085245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,64,128,1,float16,fp8,0,0.18279999494552612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,64,0,1,float16,float16,0,0.18898133436838785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,64,128,1,fp8,fp8,0,0.2321173350016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,64,0,1,float16,fp8,0,0.22665599981943765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,64,0,1,fp8,fp8,0,0.19909334182739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,64,128,1,float16,float16,0,0.1613599956035614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,64,128,1,float16,fp8,0,0.1612266699473063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,64,128,1,fp8,fp8,0,0.21506667137145996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,64,0,1,float16,fp8,0,0.18810133139292398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,64,0,1,fp8,fp8,0,0.18250133593877158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,64,128,1,float16,float16,0,0.16145599881807962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,64,0,1,float16,float16,0,0.1890986760457357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,64,0,1,fp8,fp8,0,0.18216000000635782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,64,128,1,float16,fp8,0,0.16133866707483926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,64,128,1,fp8,fp8,0,0.21361066897710165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,64,0,1,float16,fp8,0,0.19006399313608804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,64,128,1,float16,float16,0,0.16375999649365744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,64,0,1,float16,float16,0,0.189903994401296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,64,128,1,float16,float16,0,0.16613333423932394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,64,128,1,float16,fp8,0,0.16160533825556436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,64,128,1,fp8,fp8,0,0.21447465817133585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,64,0,1,float16,fp8,0,0.18957332770029703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,64,0,1,fp8,fp8,0,0.1841599941253662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,64,0,1,fp8,fp8,0,0.1832746664683024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,64,0,1,float16,float16,0,0.1928000052769979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,64,128,1,float16,fp8,0,0.16386133432388306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,64,128,1,fp8,fp8,0,0.21701333920160928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,64,0,1,float16,fp8,0,0.191594660282135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,64,128,1,float16,float16,0,0.12517333030700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,64,128,1,float16,float16,0,0.11974400281906128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,64,0,1,float16,float16,0,0.13829867045084634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,64,128,1,float16,fp8,0,0.12471466263135274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,64,128,1,fp8,fp8,0,0.15376533071200052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,64,0,1,float16,fp8,0,0.13758400082588196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,64,0,1,fp8,fp8,0,0.12903466820716858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,64,0,1,float16,float16,0,0.13180266817410788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,64,128,1,float16,fp8,0,0.11969066659609477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,64,128,1,float16,fp8,0,0.11957866946856181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,64,128,1,fp8,fp8,0,0.14274666706720987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,64,0,1,float16,fp8,0,0.13170666495958963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,64,0,1,fp8,fp8,0,0.12587733070055643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,64,128,1,float16,float16,0,0.12024533748626709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,64,128,1,float16,float16,0,0.11947733163833618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,64,0,1,float16,float16,0,0.13155200084050497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,64,128,1,fp8,fp8,0,0.14357866843541464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,64,0,1,float16,fp8,0,0.13165866335233053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,64,0,1,fp8,fp8,0,0.1268160045146942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,64,0,1,float16,float16,0,0.13220799962679544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,64,128,1,float16,fp8,0,0.11972266435623169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,64,128,1,fp8,fp8,0,0.14333867033322653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,64,0,1,float16,fp8,0,0.13147200147310892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,64,0,1,fp8,fp8,0,0.12685867150624594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,64,0,1,float16,fp8,0,0.13194666306177774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,64,0,1,fp8,fp8,0,0.12693333625793457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,64,128,1,float16,float16,0,0.12134400010108948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,64,0,1,float16,float16,0,0.13262933492660522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,64,128,1,float16,fp8,0,0.1206666628519694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,64,128,1,fp8,fp8,0,0.1455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,64,128,1,float16,float16,0,4.6864213943481445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,64,128,1,float16,fp8,0,4.708181381225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,64,0,1,fp8,fp8,0,4.880330721537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,64,128,1,float16,float16,0,4.756218592325847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,64,0,1,float16,float16,0,5.409514745076497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,64,0,1,float16,fp8,0,5.384960174560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,64,0,1,float16,float16,0,5.467109044392903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,64,128,1,fp8,fp8,0,6.184607823689778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,64,128,1,float16,fp8,0,4.6987199783325195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,64,128,1,float16,float16,0,4.763434727986653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,64,0,1,fp8,fp8,0,4.901503880818685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,64,0,1,float16,fp8,0,5.419402440388997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,64,128,1,float16,fp8,0,4.755797386169434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,64,0,1,float16,float16,0,5.444362640380859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,64,128,1,fp8,fp8,0,6.246351877848308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,64,128,1,fp8,fp8,0,6.258319854736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,64,0,1,float16,fp8,0,5.484005610148112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,64,0,1,fp8,fp8,0,4.914373397827148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,64,128,1,float16,float16,0,4.764026641845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,64,128,1,float16,fp8,0,4.77570120493571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,64,0,1,float16,float16,0,5.51252810160319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,64,128,1,fp8,fp8,0,6.312927881876628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,64,0,1,float16,fp8,0,5.488517125447591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,64,128,1,float16,float16,0,2.9754985173543296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,64,0,1,float16,float16,0,3.3182827631632485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,64,128,1,float16,fp8,0,2.8762985865275064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,64,0,1,fp8,fp8,0,4.959984143575032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,64,128,1,fp8,fp8,0,3.541221300760905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,64,128,1,float16,float16,0,2.299210707346598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,64,0,1,float16,fp8,0,3.2338879903157554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,64,0,1,fp8,fp8,0,2.870800018310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,64,0,1,float16,float16,0,2.6644585927327475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,64,128,1,float16,fp8,0,2.300704002380371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,64,128,1,fp8,fp8,0,3.0767361323038735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,64,0,1,fp8,fp8,0,2.403893311818441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,64,0,1,float16,fp8,0,2.6747360229492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,64,128,1,float16,float16,0,2.3244959513346353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,64,128,1,float16,fp8,0,2.3548107147216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,64,0,1,float16,float16,0,2.6613759994506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,64,0,1,float16,fp8,0,2.6595253944396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,64,128,1,fp8,fp8,0,3.1123412450154624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,64,0,1,fp8,fp8,0,2.4142239888509116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,64,128,1,float16,float16,0,2.3457279205322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,64,128,1,float16,fp8,0,2.3478612899780273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,64,0,1,float16,float16,0,2.705082575480143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,64,0,1,float16,fp8,0,2.705162684122721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,64,0,1,fp8,fp8,0,2.423936049143473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,64,128,1,fp8,fp8,0,3.1353813807169595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,64,128,1,float16,float16,0,2.3699413935343423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,64,0,1,float16,float16,0,2.708949406941732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,64,128,1,float16,fp8,0,2.3844426472981772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,64,128,1,float16,float16,0,1.486954689025879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,64,0,1,float16,fp8,0,2.738586743672689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,64,128,1,float16,fp8,0,1.4459039370218914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,64,128,1,fp8,fp8,0,3.137855847676595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,64,0,1,float16,float16,0,1.6885706583658855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,64,0,1,fp8,fp8,0,2.4727306365966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,64,0,1,float16,fp8,0,1.6224427223205566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,64,128,1,fp8,fp8,0,1.7786506017049153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,64,128,1,float16,float16,0,1.1644213199615479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,64,0,1,fp8,fp8,0,1.4167413711547852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,64,128,1,float16,fp8,0,1.1671040058135986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,64,0,1,float16,float16,0,1.3442079226175945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,64,0,1,float16,fp8,0,1.3868692715962727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,64,128,1,fp8,fp8,0,1.598416010538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,64,0,1,fp8,fp8,0,1.2405866781870525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,64,128,1,float16,float16,0,1.1779786745707195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,64,0,1,float16,float16,0,1.348464012145996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,64,128,1,float16,fp8,0,1.1771093209584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,64,0,1,float16,fp8,0,1.3599413235982258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,64,128,1,fp8,fp8,0,1.5545226732889812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,64,0,1,fp8,fp8,0,1.2482666969299316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,64,128,1,float16,float16,0,1.1986292997996013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,64,128,1,float16,fp8,0,1.1805813312530518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,64,0,1,float16,float16,0,1.3890132904052734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,64,0,1,float16,fp8,0,1.3484907150268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,64,128,1,fp8,fp8,0,1.558901309967041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,64,0,1,fp8,fp8,0,1.222826639811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,64,128,1,float16,float16,0,1.188047965367635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,64,128,1,float16,fp8,0,1.207370678583781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,64,0,1,float16,float16,0,1.3800586064656575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,64,128,1,float16,float16,0,0.7463200092315674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,64,0,1,float16,fp8,0,1.379151980082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,64,128,1,fp8,fp8,0,1.5758399963378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,64,0,1,float16,float16,0,0.8341066837310791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,64,0,1,fp8,fp8,0,1.225162665049235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,64,128,1,float16,fp8,0,0.7256480058034261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,64,128,1,fp8,fp8,0,0.9027093251546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,64,0,1,float16,fp8,0,0.8190666834513346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,64,128,1,float16,float16,0,0.6095733245213827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,64,0,1,fp8,fp8,0,0.7144959767659506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,64,0,1,float16,float16,0,0.6836106777191162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,64,128,1,float16,fp8,0,0.6034080187479655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,64,0,1,float16,fp8,0,0.6861066818237305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,64,128,1,fp8,fp8,0,0.7935466766357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,64,128,1,float16,float16,0,0.6116693417231241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,64,0,1,fp8,fp8,0,0.6326506535212199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,64,0,1,float16,float16,0,0.6998720169067383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,64,128,1,float16,fp8,0,0.6051520109176636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,64,128,1,fp8,fp8,0,0.7954453627268473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,64,0,1,float16,fp8,0,0.7084426879882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,64,0,1,fp8,fp8,0,0.6170080105463663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,64,128,1,float16,float16,0,0.6202506621678671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,64,0,1,float16,float16,0,0.6990880171457926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,64,128,1,float16,fp8,0,0.620901346206665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,64,128,1,fp8,fp8,0,0.8082079887390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,64,0,1,float16,fp8,0,0.69158935546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,64,0,1,fp8,fp8,0,0.6166933377583822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,64,128,1,float16,float16,0,0.6245226860046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,64,0,1,float16,float16,0,0.6958080132802328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,64,128,1,float16,fp8,0,0.6254133383433024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,64,128,1,fp8,fp8,0,0.8068373203277588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,64,0,1,float16,fp8,0,0.7113119761149088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,64,0,1,fp8,fp8,0,0.6258399883906046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,64,128,1,float16,float16,0,0.3848160107930501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,64,0,1,float16,float16,0,0.4293653170267741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,64,128,1,float16,fp8,0,0.38095998764038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,64,0,1,float16,fp8,0,0.4225546518961589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,64,128,1,fp8,fp8,0,0.47033600012461346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,64,0,1,fp8,fp8,0,0.3677440087000529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,64,128,1,float16,float16,0,0.31760533650716144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,64,0,1,float16,float16,0,0.3620320161183675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,64,128,1,float16,fp8,0,0.31522132953008014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,64,128,1,fp8,fp8,0,0.4242773453394572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,64,0,1,float16,fp8,0,0.3620320161183675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,64,0,1,fp8,fp8,0,0.3290986617406209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,64,128,1,fp8,fp8,0,0.4240479866663615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,64,128,1,float16,float16,0,0.32155734300613403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,64,0,1,float16,float16,0,0.3649119933446248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,64,128,1,float16,fp8,0,0.318725327650706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,64,0,1,float16,fp8,0,0.36692265669504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,64,0,1,fp8,fp8,0,0.3261760075887044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,64,128,1,float16,float16,0,0.32281599442164105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,64,0,1,float16,float16,0,0.3652426799138387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,64,128,1,float16,fp8,0,0.32215466101964313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,64,128,1,fp8,fp8,0,0.4227946599324544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,64,0,1,float16,fp8,0,0.36578667163848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,64,0,1,fp8,fp8,0,0.3291040062904358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,64,128,1,float16,float16,0,0.32214399178822833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,64,0,1,float16,float16,0,0.3693813482920329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,64,128,1,float16,fp8,0,0.3226240078608195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,64,128,1,fp8,fp8,0,0.4279786745707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,64,0,1,float16,fp8,0,0.36817065874735516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,64,0,1,fp8,fp8,0,0.32871999343236286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,64,128,1,float16,float16,0,0.20788800716400146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,64,0,1,float16,float16,0,0.22916799783706665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,64,128,1,float16,fp8,0,0.20435200134913126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,64,128,1,fp8,fp8,0,0.2530720035235087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,64,0,1,float16,fp8,0,0.22619199752807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,64,0,1,fp8,fp8,0,0.19338667392730713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,64,128,1,float16,float16,0,0.1730560064315796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,64,0,1,float16,float16,0,0.19669334093729654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,64,128,1,float16,fp8,0,0.17405333121617636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,64,128,1,fp8,fp8,0,0.23265600204467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,64,0,1,float16,fp8,0,0.19745065768559775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,64,0,1,fp8,fp8,0,0.1723626653353373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,64,128,1,float16,float16,0,0.17546667655309042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,64,128,1,float16,float16,0,0.17452800273895264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,64,0,1,float16,float16,0,0.1997013290723165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,64,0,1,float16,float16,0,0.19952533642450967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,64,128,1,float16,fp8,0,0.17466666301091513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,64,0,1,float16,fp8,0,0.19965867201487222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,64,128,1,fp8,fp8,0,0.23270400365193686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,64,0,1,float16,fp8,0,0.19980265696843466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,64,0,1,fp8,fp8,0,0.1742560068766276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,64,128,1,float16,fp8,0,0.17614400386810303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,64,128,1,fp8,fp8,0,0.2342346707979838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,64,0,1,float16,fp8,0,0.20132267475128174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,64,0,1,fp8,fp8,0,0.17417067289352417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,64,128,1,float16,float16,0,0.1760800083478292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,64,0,1,float16,float16,0,0.2002133329709371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,64,128,1,float16,fp8,0,0.17801066239674887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,64,128,1,fp8,fp8,0,0.23318399985631308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,64,0,1,fp8,fp8,0,0.17534399032592773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,64,128,1,float16,float16,0,0.11586133639017741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,64,0,1,float16,float16,0,0.12435199817021687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,64,128,1,float16,fp8,0,0.11481600006421407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,64,128,1,float16,fp8,0,0.09995200236638387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,64,128,1,fp8,fp8,0,0.1454026699066162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,64,0,1,float16,fp8,0,0.12110400199890137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,64,0,1,fp8,fp8,0,0.10748266180356343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,64,128,1,float16,float16,0,0.09794666369756062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,64,0,1,float16,float16,0,0.1049013336499532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,64,128,1,fp8,fp8,0,0.13345600167910257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,64,128,1,fp8,fp8,0,0.13366933663686117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,64,0,1,float16,fp8,0,0.1053013304869334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,64,0,1,fp8,fp8,0,0.09602666894594829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,64,128,1,float16,float16,0,0.09904533624649048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,64,0,1,float16,float16,0,0.10523200035095215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,64,128,1,float16,fp8,0,0.10016533732414246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,64,0,1,float16,fp8,0,0.10614933570226033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,64,0,1,fp8,fp8,0,0.09621333082516988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,64,128,1,float16,float16,0,0.10042132933934529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,64,0,1,fp8,fp8,0,0.09777067104975383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,64,128,1,float16,float16,0,0.09969600041707356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,64,128,1,float16,fp8,0,0.10104533036549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,64,0,1,float16,float16,0,0.10616000493367513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,64,128,1,float16,fp8,0,0.10095466176668803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,64,128,1,fp8,fp8,0,0.13379733761151633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,64,0,1,float16,fp8,0,0.10694400469462077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,64,0,1,float16,float16,0,0.10658133029937744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,64,128,1,fp8,fp8,0,0.13661866386731467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,64,0,1,float16,fp8,0,0.10680533448855083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,64,0,1,fp8,fp8,0,0.09753066301345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,64,128,1,float16,float16,0,0.06614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,64,0,1,float16,float16,0,0.06769066552321117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,64,128,1,float16,fp8,0,0.06542933483918507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,64,128,1,fp8,fp8,0,0.08806932965914409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,64,0,1,float16,fp8,0,0.06602666775385539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,64,0,1,fp8,fp8,0,0.06259199976921082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,64,128,1,float16,float16,0,0.058517331878344216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,64,0,1,float16,float16,0,0.05961066484451294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,64,128,1,float16,fp8,0,0.05841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,64,128,1,fp8,fp8,0,0.0757173349459966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,64,128,1,float16,fp8,0,0.05909866591294607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,64,0,1,float16,fp8,0,0.05975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,64,0,1,fp8,fp8,0,0.055573334296544395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,64,128,1,float16,float16,0,0.059434667229652405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,64,0,1,float16,float16,0,0.06031466523806254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,64,128,1,fp8,fp8,0,0.07493866483370464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,64,0,1,float16,fp8,0,0.06118933359781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,64,0,1,fp8,fp8,0,0.05574400226275126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,64,128,1,float16,float16,0,0.0589279979467392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,64,0,1,fp8,fp8,0,0.05644266804059347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,64,0,1,float16,float16,0,0.06083733340104421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,64,128,1,float16,fp8,0,0.06097066899140676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,64,128,1,fp8,fp8,0,0.0751093327999115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,64,0,1,float16,fp8,0,0.06166933476924896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,64,128,1,float16,float16,0,0.0603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,64,0,1,float16,float16,0,0.06155199805895487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,64,128,1,float16,fp8,0,0.059903999169667564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,64,128,1,fp8,fp8,0,0.07779733339945476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,64,0,1,float16,fp8,0,0.06145066519578298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,64,0,1,fp8,fp8,0,0.055914665261904396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,64,128,1,float16,float16,0,4.43502934773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,64,0,1,float16,float16,0,4.396970748901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,64,128,1,float16,fp8,0,4.445760091145833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,64,0,1,fp8,fp8,0,3.953887939453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,64,0,1,float16,fp8,0,4.4253387451171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,64,128,1,float16,float16,0,4.489402770996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,64,128,1,fp8,fp8,0,5.884101231892903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,64,0,1,float16,float16,0,4.463552157084147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,64,128,1,float16,fp8,0,4.459247907002767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,64,0,1,fp8,fp8,0,3.9907732009887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,64,0,1,float16,fp8,0,4.475882530212402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,64,128,1,float16,float16,0,4.5152587890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,64,128,1,fp8,fp8,0,5.9040266672770185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,64,0,1,float16,float16,0,4.499130566914876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,64,128,1,float16,fp8,0,4.541952133178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,64,0,1,fp8,fp8,0,3.996368090311686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,64,0,1,float16,fp8,0,4.499914805094401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,64,128,1,float16,float16,0,4.518437385559082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,64,0,1,float16,float16,0,4.523877461751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,64,128,1,fp8,fp8,0,5.924421310424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,64,128,1,float16,fp8,0,4.500778516133626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,64,0,1,float16,fp8,0,4.541893323262532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,64,128,1,float16,float16,0,2.8491252263387046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,64,0,1,float16,float16,0,2.818826675415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,64,128,1,fp8,fp8,0,5.944133122762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,64,0,1,fp8,fp8,0,4.039658546447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,64,128,1,float16,fp8,0,2.743472099304199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,64,0,1,float16,fp8,0,2.727317492167155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,64,128,1,fp8,fp8,0,3.3495734532674155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,64,0,1,fp8,fp8,0,2.383018652598063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,64,128,1,float16,float16,0,2.185541311899821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,64,0,1,float16,float16,0,2.151365280151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,64,128,1,float16,fp8,0,2.166869322458903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,64,0,1,float16,fp8,0,2.155893325805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,64,0,1,fp8,fp8,0,1.9224106470743816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,64,128,1,fp8,fp8,0,2.901621182759603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,64,128,1,float16,float16,0,2.197183926900228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,64,0,1,float16,float16,0,2.1847146352132163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,64,128,1,float16,fp8,0,2.206266721089681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,64,0,1,float16,fp8,0,2.1825332641601562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,64,0,1,fp8,fp8,0,1.952847957611084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,64,128,1,fp8,fp8,0,2.914896011352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,64,128,1,float16,float16,0,2.19484806060791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,64,0,1,float16,float16,0,2.209717273712158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,64,128,1,float16,fp8,0,2.2207840283711753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,64,0,1,float16,fp8,0,2.2149012883504233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,64,128,1,fp8,fp8,0,2.9357067743937173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,64,0,1,fp8,fp8,0,1.960576057434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,64,128,1,float16,float16,0,2.2467254002889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,64,128,1,float16,fp8,0,2.2123252550760903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,64,0,1,float16,float16,0,2.2069013913472495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,64,0,1,float16,fp8,0,2.2223946253458657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,64,128,1,fp8,fp8,0,2.959199905395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,64,128,1,float16,float16,0,1.426645278930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,64,0,1,fp8,fp8,0,2.006399949391683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,64,0,1,float16,float16,0,1.41102933883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,64,128,1,fp8,fp8,0,1.67031462987264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,64,128,1,float16,fp8,0,1.3793439865112305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,64,0,1,float16,fp8,0,1.3677760759989421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,64,0,1,fp8,fp8,0,1.1858879725138347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,64,128,1,float16,float16,0,1.1013973553975422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,64,0,1,float16,float16,0,1.088543971379598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,64,128,1,float16,fp8,0,1.1360960006713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,64,0,1,float16,fp8,0,1.087381362915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,64,0,1,fp8,fp8,0,0.9695040384928385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,64,128,1,fp8,fp8,0,1.4999359448750813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,64,128,1,float16,float16,0,1.1106506983439128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,64,0,1,float16,float16,0,1.1044159730275471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,64,128,1,float16,fp8,0,1.1120479901631672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,64,0,1,float16,fp8,0,1.1303093433380127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,64,128,1,fp8,fp8,0,1.4675413767496746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,64,0,1,fp8,fp8,0,0.9678986867268881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,64,128,1,float16,float16,0,1.1198879877726238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,64,0,1,float16,float16,0,1.1204480330149333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,64,128,1,float16,fp8,0,1.113653341929118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,64,0,1,fp8,fp8,0,0.9947199821472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,64,0,1,float16,fp8,0,1.1009066899617512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,64,128,1,fp8,fp8,0,1.4741120338439941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,64,128,1,float16,float16,0,1.1231520175933838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,64,0,1,float16,float16,0,1.1093119780222576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,64,128,1,float16,fp8,0,1.135317325592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,64,128,1,float16,float16,0,0.7187946637471517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,64,0,1,float16,fp8,0,1.1078773339589436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,64,0,1,fp8,fp8,0,0.9938613573710123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,64,0,1,float16,float16,0,0.7071786721547445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,64,128,1,fp8,fp8,0,1.5057652791341145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,64,128,1,float16,fp8,0,0.6981866359710693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,64,128,1,fp8,fp8,0,0.8509120146433512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,64,128,1,float16,float16,0,0.5663359959920248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,64,0,1,float16,fp8,0,0.6874293486277262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,64,0,1,fp8,fp8,0,0.5885119835535685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,64,0,1,float16,float16,0,0.5638186534245809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,64,128,1,float16,fp8,0,0.576197346051534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,64,128,1,fp8,fp8,0,0.7516960302988688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,64,0,1,float16,fp8,0,0.5639520088831583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,64,0,1,fp8,fp8,0,0.49764267603556317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,64,128,1,float16,float16,0,0.5745600064595541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,64,128,1,fp8,fp8,0,0.7593493461608887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,64,0,1,float16,float16,0,0.566213329633077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,64,0,1,float16,fp8,0,0.568618655204773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,64,128,1,float16,fp8,0,0.5781813462575277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,64,0,1,fp8,fp8,0,0.49929598967234295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,64,128,1,float16,float16,0,0.5752533276875814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,64,0,1,float16,float16,0,0.5707573493321737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,64,128,1,float16,fp8,0,0.5806613365809122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,64,0,1,float16,fp8,0,0.5703146855036417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,64,0,1,fp8,fp8,0,0.5037173430124918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,64,128,1,fp8,fp8,0,0.7621653079986572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,64,128,1,float16,float16,0,0.5843679904937744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,64,0,1,float16,float16,0,0.5728906790415446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,64,128,1,float16,fp8,0,0.5854293505350748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,64,128,1,fp8,fp8,0,0.7631093660990397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,64,0,1,float16,fp8,0,0.574127991994222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,64,128,1,float16,float16,0,0.37323200702667236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,64,0,1,fp8,fp8,0,0.5056266784667969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,64,0,1,float16,float16,0,0.3652160167694092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,64,128,1,float16,fp8,0,0.36087465286254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,64,128,1,fp8,fp8,0,0.4392106533050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,64,0,1,float16,fp8,0,0.3548053503036499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,64,0,1,fp8,fp8,0,0.3039199908574422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,64,128,1,float16,float16,0,0.30209600925445557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,64,0,1,float16,float16,0,0.2943519949913025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,64,128,1,float16,fp8,0,0.30339733759562176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,64,128,1,fp8,fp8,0,0.39794135093688965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,64,0,1,float16,fp8,0,0.2943039933840434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,64,0,1,fp8,fp8,0,0.2635413408279419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,64,128,1,float16,float16,0,0.303056001663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,64,0,1,float16,float16,0,0.2972799936930339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,64,128,1,float16,fp8,0,0.30501333872477215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,64,128,1,fp8,fp8,0,0.39800000190734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,64,0,1,float16,fp8,0,0.299562672773997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,64,0,1,fp8,fp8,0,0.2646080056826274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,64,128,1,float16,float16,0,0.30508800347646076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,64,0,1,float16,float16,0,0.2978293299674988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,64,128,1,float16,fp8,0,0.3046879967053731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,64,128,1,fp8,fp8,0,0.3991039991378784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,64,0,1,float16,fp8,0,0.2983040014902751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,64,0,1,fp8,fp8,0,0.26502933104832965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,64,128,1,fp8,fp8,0,0.4002773364384969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,64,128,1,float16,float16,0,0.30899200836817425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,64,0,1,float16,float16,0,0.30162666241327923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,64,128,1,float16,fp8,0,0.30771199862162274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,64,0,1,float16,fp8,0,0.30083733797073364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,64,128,1,fp8,fp8,0,0.2379200061162313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,64,0,1,fp8,fp8,0,0.2670186758041382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,64,128,1,float16,float16,0,0.19925866524378458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,64,0,1,float16,float16,0,0.19357866048812866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,64,0,1,float16,float16,0,0.16085333625475565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,64,128,1,float16,fp8,0,0.1941439906756083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,64,0,1,float16,fp8,0,0.1895093321800232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,64,0,1,fp8,fp8,0,0.1437173287073771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,64,0,1,fp8,fp8,0,0.16342399517695108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,64,128,1,float16,float16,0,0.16459199786186218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,64,128,1,float16,fp8,0,0.16609600186347961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,64,128,1,fp8,fp8,0,0.21805334091186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,64,0,1,float16,fp8,0,0.16061866283416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,64,0,1,float16,fp8,0,0.16217066844304404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,64,128,1,float16,float16,0,0.16595199704170227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,64,0,1,float16,float16,0,0.16200000047683716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,64,128,1,float16,fp8,0,0.1678666671117147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,64,128,1,fp8,fp8,0,0.21944000323613486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,64,128,1,fp8,fp8,0,0.2193066676457723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,64,0,1,float16,fp8,0,0.16160533825556436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,64,0,1,fp8,fp8,0,0.144186665614446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,64,128,1,float16,float16,0,0.1665226618448893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,64,0,1,float16,float16,0,0.16244799892107645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,64,128,1,float16,fp8,0,0.16726932922999063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,64,0,1,fp8,fp8,0,0.14453333616256714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,64,128,1,float16,float16,0,0.16909333070119223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,64,0,1,float16,float16,0,0.16326399644215903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,64,128,1,float16,fp8,0,0.16856533288955688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,64,128,1,fp8,fp8,0,0.2207146684328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,64,0,1,float16,fp8,0,0.1637279987335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,64,0,1,fp8,fp8,0,0.14656000336011252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,64,128,1,float16,float16,0,0.11133866508801778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,64,0,1,float16,float16,0,0.11309333642323811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,64,128,1,float16,fp8,0,0.10986666878064473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,64,128,1,fp8,fp8,0,0.13870400190353394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,64,0,1,float16,fp8,0,0.10975999633471172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,64,0,1,fp8,fp8,0,0.09179733196894328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,64,128,1,float16,float16,0,0.09408000111579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,64,0,1,float16,float16,0,0.08872532844543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,64,128,1,float16,fp8,0,0.09342400232950847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,64,128,1,fp8,fp8,0,0.12473066647847493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,64,0,1,float16,fp8,0,0.08897067109743755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,64,0,1,fp8,fp8,0,0.08122133215268452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,64,128,1,float16,float16,0,0.0941546658674876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,64,0,1,fp8,fp8,0,0.08116266628106435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,64,0,1,float16,float16,0,0.0895306666692098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,64,128,1,float16,fp8,0,0.09604266285896301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,64,128,1,fp8,fp8,0,0.12571199735005698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,64,0,1,float16,fp8,0,0.09006933371225993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,64,0,1,fp8,fp8,0,0.08155733346939087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,64,128,1,float16,float16,0,0.09490133325258891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,64,0,1,float16,float16,0,0.08853333195050557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,64,128,1,float16,fp8,0,0.09659199913342793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,64,128,1,fp8,fp8,0,0.12689066926638284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,64,0,1,float16,fp8,0,0.09060800075531006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,64,128,1,float16,float16,0,0.0959999958674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,64,0,1,float16,float16,0,0.09096533060073853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,64,128,1,float16,fp8,0,0.09573333462079366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,64,128,1,fp8,fp8,0,0.12707199652989706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,64,0,1,float16,fp8,0,0.09101866682370503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,64,128,1,fp8,fp8,0,0.0837653378645579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,64,0,1,fp8,fp8,0,0.08297066887219746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,64,128,1,float16,float16,0,0.0634986658891042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,64,0,1,float16,float16,0,0.05888533095518748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,64,128,1,float16,fp8,0,0.06247999767462412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,64,0,1,float16,fp8,0,0.05819733440876007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,64,0,1,fp8,fp8,0,0.054117331902186074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,64,128,1,float16,float16,0,0.05528533458709717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,64,0,1,float16,float16,0,0.05157333115736643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,64,128,1,float16,fp8,0,0.05634133517742157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,64,128,1,fp8,fp8,0,0.07287999987602234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,64,0,1,float16,fp8,0,0.0516533354918162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,64,0,1,fp8,fp8,0,0.04709866642951965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,64,128,1,float16,float16,0,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,64,0,1,float16,float16,0,0.05201066533724467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,64,128,1,float16,fp8,0,0.05644266804059347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,64,128,1,fp8,fp8,0,0.07266133526961009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,64,0,1,float16,fp8,0,0.05213333169619242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,64,0,1,fp8,fp8,0,0.04683200021584829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,64,128,1,float16,float16,0,0.05730666716893514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,64,0,1,float16,float16,0,0.05268799761931101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,64,128,1,float16,fp8,0,0.0569653312365214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,64,128,1,fp8,fp8,0,0.07411199808120728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,64,0,1,float16,fp8,0,0.05282666782538096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,64,0,1,fp8,fp8,0,0.04751466711362203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,64,128,1,float16,float16,0,0.0566293348868688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,64,0,1,float16,float16,0,0.05327466626962026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,64,128,1,float16,fp8,0,0.05740800003210703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,64,128,1,fp8,fp8,0,0.07317333420117696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,64,0,1,float16,fp8,0,0.05277333160241445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,64,0,1,float16,fp8,0,0.03603733330965042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,64,0,1,fp8,fp8,0,0.04753066599369049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,64,128,1,float16,float16,0,0.041264000038305916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,64,0,1,float16,float16,0,0.03583466758330663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,64,128,1,float16,fp8,0,0.041738669077555336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,64,128,1,fp8,fp8,0,0.05189333359400431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,64,0,1,fp8,fp8,0,0.031983998914559685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,64,128,1,float16,float16,0,0.038319999972979225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,64,0,1,float16,float16,0,0.0329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,64,128,1,float16,fp8,0,0.03928533444801966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,64,128,1,fp8,fp8,0,0.04952000081539154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,64,0,1,float16,fp8,0,0.03305066625277201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,64,0,1,fp8,fp8,0,0.0305226668715477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,64,128,1,float16,float16,0,0.03962666789690653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,64,0,1,float16,float16,0,0.03282133241494497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,64,128,1,float16,fp8,0,0.03938666731119156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,64,128,1,fp8,fp8,0,0.05014933149019877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,64,0,1,float16,fp8,0,0.033413333197434746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,64,0,1,fp8,fp8,0,0.030853333572546642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,64,128,1,float16,float16,0,0.03905066599448522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,64,0,1,float16,float16,0,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,64,128,1,float16,fp8,0,0.040037333965301514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,64,128,1,fp8,fp8,0,0.05013866722583771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,64,0,1,float16,fp8,0,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,64,0,1,fp8,fp8,0,0.030453334252039593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,64,128,1,float16,float16,0,0.040074666341145836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,64,0,1,float16,float16,0,0.0341333324710528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,64,128,1,float16,fp8,0,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,64,128,1,fp8,fp8,0,0.04994133114814758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,64,0,1,float16,fp8,0,0.03422933320204417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,64,0,1,fp8,fp8,0,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,64,128,1,float16,float16,0,1.8857119878133137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,64,0,1,float16,float16,0,1.829429308573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,64,128,1,float16,fp8,0,1.883994738260905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,64,0,1,fp8,fp8,0,1.6953973770141602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,64,0,1,float16,fp8,0,1.8303573926289876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,64,128,1,float16,float16,0,1.8954826990763347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,64,0,1,float16,float16,0,1.8513013521830242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,64,128,1,fp8,fp8,0,2.581461270650228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,64,128,1,float16,fp8,0,1.8914613723754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,64,0,1,float16,fp8,0,1.8299360275268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,64,0,1,fp8,fp8,0,1.7016960779825847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,64,128,1,fp8,fp8,0,2.5885866483052573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,64,128,1,float16,float16,0,1.9418026606241863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,64,0,1,float16,float16,0,1.8984586397806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,64,128,1,float16,fp8,0,1.9111040433247883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,64,128,1,fp8,fp8,0,2.583738644917806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,64,0,1,float16,fp8,0,1.885584036509196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,64,0,1,fp8,fp8,0,1.7100906372070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,64,128,1,float16,float16,0,1.9498987197875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,64,0,1,float16,float16,0,1.9377600351969402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,64,128,1,float16,fp8,0,1.9684213002522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,64,128,1,float16,float16,0,1.270250638326009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,64,128,1,fp8,fp8,0,2.6028800010681152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,64,0,1,float16,fp8,0,1.8711999257405598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,64,0,1,fp8,fp8,0,1.7256959279378254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,64,128,1,float16,fp8,0,1.2180639902750652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,64,0,1,float16,float16,0,1.2404853502909343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,64,128,1,fp8,fp8,0,1.4954773585001628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,64,0,1,fp8,fp8,0,1.0324640274047852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,64,0,1,float16,fp8,0,1.1996853351593018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,64,128,1,float16,float16,0,0.9311467011769613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,64,0,1,float16,float16,0,0.931493361790975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,64,128,1,float16,fp8,0,0.9574879805246989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,64,0,1,float16,fp8,0,0.921295960744222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,64,0,1,fp8,fp8,0,0.83897598584493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,64,128,1,fp8,fp8,0,1.3031893571217854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,64,128,1,float16,float16,0,0.9620532989501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,64,0,1,float16,float16,0,0.9132426579793295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,64,128,1,float16,fp8,0,0.9631253083546957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,64,0,1,fp8,fp8,0,0.8408532937367758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,64,0,1,float16,fp8,0,0.9272053241729736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,64,128,1,fp8,fp8,0,1.3082613150278728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,64,128,1,float16,float16,0,0.9568320115407308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,64,0,1,float16,float16,0,0.9438827037811279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,64,128,1,float16,fp8,0,0.9429706732432047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,64,128,1,fp8,fp8,0,1.2940479914347331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,64,0,1,float16,fp8,0,0.9248639742533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,64,0,1,fp8,fp8,0,0.8571840127309164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,64,128,1,float16,float16,0,0.9635146458943685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,64,0,1,float16,float16,0,0.9384480317433676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,64,128,1,float16,fp8,0,0.9748160044352213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,64,0,1,float16,fp8,0,0.9401813348134359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,64,128,1,fp8,fp8,0,1.289952039718628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,64,128,1,float16,float16,0,0.6361013253529867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,64,0,1,float16,float16,0,0.6234186490376791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,64,0,1,fp8,fp8,0,0.868559996287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,64,128,1,float16,fp8,0,0.6128213405609131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,64,0,1,float16,fp8,0,0.6029813289642334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,64,128,1,fp8,fp8,0,0.7364106973012289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,64,0,1,fp8,fp8,0,0.5172266562779745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,64,128,1,float16,float16,0,0.4782986640930176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,64,0,1,float16,float16,0,0.46809065341949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,64,128,1,float16,fp8,0,0.48261332511901855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,64,0,1,float16,fp8,0,0.4724586804707845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,64,128,1,fp8,fp8,0,0.6537280082702637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,64,0,1,fp8,fp8,0,0.43357332547505695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,64,128,1,float16,float16,0,0.48581866423288983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,64,0,1,float16,float16,0,0.4742453495661418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,64,128,1,float16,fp8,0,0.48313601811726886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,64,128,1,fp8,fp8,0,0.6505173444747925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,64,0,1,float16,fp8,0,0.47408533096313477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,64,0,1,fp8,fp8,0,0.43271466096242267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,64,128,1,float16,float16,0,0.4884106715520223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,64,0,1,float16,float16,0,0.47514665126800537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,64,128,1,float16,fp8,0,0.48867201805114746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,64,128,1,fp8,fp8,0,0.6581706603368124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,64,0,1,float16,fp8,0,0.4703893264134725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,64,0,1,fp8,fp8,0,0.43324267864227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,64,128,1,float16,float16,0,0.49378665288289386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,64,0,1,float16,float16,0,0.48044800758361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,64,128,1,float16,fp8,0,0.4922826687494914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,64,128,1,fp8,fp8,0,0.6590079863866171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,64,0,1,float16,fp8,0,0.48068265120188397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,64,0,1,fp8,fp8,0,0.4381706714630127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,64,128,1,float16,float16,0,0.33079467217127484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,64,0,1,float16,float16,0,0.3245120048522949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,64,128,1,float16,fp8,0,0.32095466057459515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,64,128,1,fp8,fp8,0,0.37701865037282306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,64,0,1,float16,fp8,0,0.316101332505544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,64,0,1,fp8,fp8,0,0.26660799980163574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,64,128,1,float16,float16,0,0.25199999411900836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,64,0,1,float16,float16,0,0.24691200256347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,64,0,1,fp8,fp8,0,0.2293813427289327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,64,128,1,float16,float16,0,0.2542133331298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,64,128,1,float16,fp8,0,0.2515733242034912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,64,128,1,fp8,fp8,0,0.3379733165105184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,64,0,1,float16,fp8,0,0.24678399165471396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,64,0,1,float16,float16,0,0.24739199876785278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,64,128,1,float16,fp8,0,0.2546186645825704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,64,128,1,fp8,fp8,0,0.34171732266743976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,64,0,1,float16,fp8,0,0.2488373319307963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,64,0,1,fp8,fp8,0,0.22919466098149618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,64,128,1,float16,float16,0,0.2545119921366374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,64,0,1,float16,fp8,0,0.2495466669400533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,64,0,1,float16,float16,0,0.2488373319307963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,64,128,1,float16,fp8,0,0.25545066595077515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,64,128,1,fp8,fp8,0,0.34006400903066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,64,0,1,fp8,fp8,0,0.23036799828211466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,64,128,1,float16,float16,0,0.25838400920232135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,64,0,1,float16,float16,0,0.25114667415618896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,64,128,1,float16,fp8,0,0.25778667132059735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,64,128,1,fp8,fp8,0,0.342031995455424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,64,0,1,float16,fp8,0,0.2527093291282654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,64,0,1,fp8,fp8,0,0.2322453260421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,64,128,1,float16,float16,0,0.1780959963798523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,64,0,1,fp8,fp8,0,0.14673067132631937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,64,0,1,float16,float16,0,0.17585599422454834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,64,128,1,float16,fp8,0,0.17355199654897055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,64,128,1,fp8,fp8,0,0.19868799050649008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,64,0,1,float16,fp8,0,0.1706506609916687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,64,128,1,float16,float16,0,0.1347040037314097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,64,0,1,float16,float16,0,0.132533331712087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,64,128,1,float16,fp8,0,0.13546666502952576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,64,128,1,fp8,fp8,0,0.18191999197006226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,64,0,1,float16,fp8,0,0.13269866506258646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,64,0,1,fp8,fp8,0,0.12742933630943298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,64,128,1,float16,float16,0,0.1358560025691986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,64,0,1,float16,fp8,0,0.13316266735394797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,64,0,1,float16,float16,0,0.13246933619181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,64,128,1,float16,fp8,0,0.13576533397038779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,64,128,1,fp8,fp8,0,0.18031466007232666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,64,0,1,fp8,fp8,0,0.1281653344631195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,64,0,1,float16,fp8,0,0.1339040001233419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,64,128,1,float16,float16,0,0.1362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,64,0,1,float16,float16,0,0.13359999656677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,64,128,1,float16,fp8,0,0.13660800457000732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,64,128,1,fp8,fp8,0,0.181551992893219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,64,0,1,fp8,fp8,0,0.12841066718101501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,64,128,1,float16,float16,0,0.13827199737230936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,64,0,1,float16,float16,0,0.13614400227864584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,64,128,1,float16,fp8,0,0.13827199737230936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,64,128,1,fp8,fp8,0,0.1844373345375061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,64,0,1,float16,fp8,0,0.13612266381581625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,64,0,1,fp8,fp8,0,0.1297760009765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,64,128,1,float16,float16,0,0.0974026620388031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,64,0,1,fp8,fp8,0,0.08431466420491536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,64,0,1,float16,float16,0,0.09649067123730977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,64,128,1,float16,float16,0,0.07373866438865662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,64,128,1,float16,fp8,0,0.09521599610646565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,64,128,1,fp8,fp8,0,0.10936533411343892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,64,0,1,float16,fp8,0,0.09490666786829631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,64,0,1,float16,float16,0,0.07141333321730296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,64,128,1,float16,fp8,0,0.07367466886838277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,64,128,1,fp8,fp8,0,0.09894399841626485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,64,0,1,float16,fp8,0,0.07258666555086772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,64,0,1,fp8,fp8,0,0.07208000123500824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,64,128,1,float16,float16,0,0.07455466687679291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,64,0,1,float16,float16,0,0.07197333375612895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,64,128,1,float16,fp8,0,0.07355733215808868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,64,128,1,fp8,fp8,0,0.09918933113416036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,64,0,1,float16,fp8,0,0.07234666744867961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,64,0,1,fp8,fp8,0,0.0727040022611618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,64,128,1,float16,float16,0,0.07417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,64,0,1,float16,float16,0,0.0724373310804367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,64,0,1,float16,float16,0,0.07396266857783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,64,128,1,float16,fp8,0,0.07437866429487865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,64,128,1,fp8,fp8,0,0.09877333045005798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,64,0,1,float16,fp8,0,0.07266133526961009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,64,0,1,fp8,fp8,0,0.07251733541488647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,64,128,1,float16,float16,0,0.07502933343251546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,64,128,1,float16,fp8,0,0.07572799921035767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,64,128,1,fp8,fp8,0,0.0997279981772105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,64,0,1,float16,fp8,0,0.07397866745789845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,64,0,1,fp8,fp8,0,0.07328000168005626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,64,128,1,float16,float16,0,0.04799999793370565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,64,0,1,float16,float16,0,0.04746133089065552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,64,128,1,float16,fp8,0,0.04714133342107137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,64,128,1,fp8,fp8,0,0.06385066608587901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,64,0,1,float16,fp8,0,0.04650666813055674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,64,0,1,fp8,fp8,0,0.048010667165120445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,64,128,1,float16,float16,0,0.04147200038035711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,64,128,1,float16,float16,0,0.0410453329483668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,64,0,1,float16,float16,0,0.04015466570854187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,64,128,1,float16,fp8,0,0.04167999823888143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,64,128,1,float16,fp8,0,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,64,128,1,fp8,fp8,0,0.055061335364977516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,64,0,1,float16,fp8,0,0.04038933416207632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,64,0,1,fp8,fp8,0,0.04228266576925913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,64,0,1,float16,float16,0,0.040474665661652885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,64,128,1,fp8,fp8,0,0.05516799787680308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,64,0,1,float16,fp8,0,0.040448000033696495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,64,0,1,fp8,fp8,0,0.04227200150489807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,64,128,1,float16,float16,0,0.04222933451334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,64,0,1,float16,float16,0,0.040405333042144775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,64,0,1,float16,float16,0,0.04177066683769226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,64,128,1,float16,fp8,0,0.04262933135032654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,64,128,1,float16,fp8,0,0.041696002086003624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,64,128,1,fp8,fp8,0,0.05585066477457682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,64,0,1,float16,fp8,0,0.040805332362651825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,64,0,1,fp8,fp8,0,0.04195199906826019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,64,128,1,float16,float16,0,0.04147200038035711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,64,128,1,fp8,fp8,0,0.05605866511662801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,64,0,1,float16,fp8,0,0.04190400242805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,64,0,1,fp8,fp8,0,0.04330666859944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,64,128,1,float16,float16,0,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,64,0,1,float16,float16,0,0.033013333876927696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,64,128,1,float16,fp8,0,0.033557333052158356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,64,128,1,fp8,fp8,0,0.03612799942493439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,64,128,1,fp8,fp8,0,0.034517332911491394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,64,0,1,float16,fp8,0,0.03272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,64,0,1,fp8,fp8,0,0.028794666131337483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,64,128,1,float16,float16,0,0.030432000756263733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,64,0,1,float16,float16,0,0.029882666965325672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,64,128,1,float16,fp8,0,0.030559999247392017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,64,0,1,float16,fp8,0,0.030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,64,0,1,fp8,fp8,0,0.028090665737787884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,64,128,1,float16,float16,0,0.03032533327738444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,64,0,1,float16,float16,0,0.03035199890534083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,64,128,1,float16,float16,0,0.030981334547201794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,64,128,1,float16,fp8,0,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,64,128,1,fp8,fp8,0,0.03446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,64,0,1,float16,fp8,0,0.030634666482607525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,64,0,1,fp8,fp8,0,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,64,0,1,fp8,fp8,0,0.028031999866167705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,64,0,1,float16,float16,0,0.03018666555484136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,64,128,1,float16,fp8,0,0.031061333914597828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,64,128,1,fp8,fp8,0,0.03512533257404963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,64,0,1,float16,fp8,0,0.03065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,64,128,1,float16,float16,0,0.03147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,64,0,1,float16,float16,0,0.03081600119670232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,64,128,1,float16,fp8,0,0.031386665999889374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,64,128,1,fp8,fp8,0,0.0348693331082662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,64,128,1,float16,fp8,0,0.02187199890613556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,64,0,1,float16,fp8,0,0.03091199944416682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,64,0,1,fp8,fp8,0,0.02867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,64,0,1,fp8,fp8,0,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,64,128,1,float16,float16,0,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,64,0,1,float16,float16,0,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,64,128,1,fp8,fp8,0,0.024885334074497223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,64,0,1,float16,fp8,0,0.021562665700912476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,64,128,1,float16,float16,0,0.02025066688656807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,64,0,1,float16,float16,0,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,64,128,1,float16,fp8,0,0.02046400060256322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,64,128,1,fp8,fp8,0,0.024031999210516613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,64,0,1,float16,fp8,0,0.02011200040578842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,64,0,1,fp8,fp8,0,0.02088533341884613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,64,128,1,float16,float16,0,0.020234666764736176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,64,0,1,float16,float16,0,0.02040533348917961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,64,128,1,float16,fp8,0,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,64,128,1,fp8,fp8,0,0.023941333095232647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,64,0,1,float16,fp8,0,0.02027733375628789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,64,0,1,fp8,fp8,0,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,64,128,1,float16,float16,0,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,64,0,1,float16,float16,0,0.020202666521072388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,64,128,1,float16,fp8,0,0.020319999506076176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,64,128,1,fp8,fp8,0,0.024677333732446034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,64,0,1,float16,fp8,0,0.02027200038234393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,64,0,1,fp8,fp8,0,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,64,128,1,float16,float16,0,0.02027200038234393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,64,0,1,float16,float16,0,0.02032533288002014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,64,128,1,float16,fp8,0,0.020549333343903225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,64,128,1,fp8,fp8,0,0.024314666787783306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,64,0,1,float16,fp8,0,0.020245333512624104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,64,0,1,fp8,fp8,0,0.020810666183630627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,64,128,1,float16,float16,0,0.6910080115000407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,64,0,1,float16,float16,0,0.6918293635050455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,64,128,1,float16,fp8,0,0.6966079870859782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,64,128,1,fp8,fp8,0,0.8512427012125651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,64,0,1,float16,fp8,0,0.6901600360870361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,64,0,1,fp8,fp8,0,0.8451519807179769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,64,128,1,float16,float16,0,0.6966026624043783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,64,0,1,float16,float16,0,0.6978186766306559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,64,128,1,float16,fp8,0,0.6964639822642008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,64,128,1,fp8,fp8,0,0.8523093064626058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,64,0,1,float16,fp8,0,0.6962292989095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,64,0,1,fp8,fp8,0,0.8591093222300211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,64,128,1,float16,float16,0,0.7020320097605387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,64,0,1,float16,float16,0,0.7047253449757894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,64,128,1,float16,fp8,0,0.7016639709472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,64,0,1,float16,fp8,0,0.6993280251820883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,64,128,1,fp8,fp8,0,0.861290693283081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,64,128,1,float16,float16,0,0.7130400339762369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,64,0,1,fp8,fp8,0,0.84879469871521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,64,0,1,float16,float16,0,0.7118933200836182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,64,128,1,float16,fp8,0,0.7083520094553629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,64,128,1,fp8,fp8,0,0.8630399703979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,64,0,1,float16,fp8,0,0.7091573079427084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,64,128,1,float16,float16,0,0.525541345278422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,64,0,1,float16,float16,0,0.5279146830240885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,64,0,1,fp8,fp8,0,0.8580426375071207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,64,128,1,float16,fp8,0,0.5044586658477783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,64,128,1,fp8,fp8,0,0.5202186504999796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,64,128,1,float16,float16,0,0.35441064834594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,64,0,1,float16,fp8,0,0.5055679877599081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,64,0,1,fp8,fp8,0,0.5166133244832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,64,0,1,float16,float16,0,0.35231467088063556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,64,128,1,float16,fp8,0,0.3570133447647095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,64,0,1,float16,fp8,0,0.3525173266728719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,64,128,1,fp8,fp8,0,0.4301439921061198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,64,0,1,fp8,fp8,0,0.430730660756429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,64,128,1,float16,float16,0,0.3575199842453003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,64,0,1,float16,float16,0,0.354912002881368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,64,128,1,float16,fp8,0,0.35601067543029785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,64,128,1,fp8,fp8,0,0.4326666593551636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,64,0,1,float16,fp8,0,0.35920532544453937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,64,0,1,fp8,fp8,0,0.4336373408635457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,64,128,1,fp8,fp8,0,0.43190932273864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,64,128,1,float16,float16,0,0.35725335280100506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,64,0,1,float16,float16,0,0.3593653440475464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,64,128,1,float16,fp8,0,0.36366931597391766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,64,128,1,fp8,fp8,0,0.43231467405955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,64,0,1,float16,fp8,0,0.3590826590855916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,64,0,1,fp8,fp8,0,0.4331626494725545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,64,128,1,float16,float16,0,0.367082675298055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,64,0,1,float16,fp8,0,0.36367468039194745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,64,128,1,float16,float16,0,0.2731200059254964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,64,0,1,float16,float16,0,0.3654239972432454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,64,128,1,float16,fp8,0,0.36302932103474933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,64,0,1,fp8,fp8,0,0.43885334332784015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,64,0,1,float16,float16,0,0.2737813393274943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,64,128,1,float16,fp8,0,0.2643360098203023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,64,128,1,fp8,fp8,0,0.26586665709813434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,64,0,1,float16,fp8,0,0.2630293369293213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,64,0,1,fp8,fp8,0,0.2661973237991333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,64,128,1,float16,float16,0,0.18409599860509238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,64,0,1,float16,float16,0,0.18623467286427817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,64,128,1,float16,fp8,0,0.18571732441584268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,64,128,1,fp8,fp8,0,0.23105067014694214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,64,0,1,float16,fp8,0,0.18466667334238687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,64,0,1,fp8,fp8,0,0.22745599349339804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,64,128,1,float16,float16,0,0.18622400363286337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,64,0,1,float16,float16,0,0.18557866414388022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,64,128,1,float16,fp8,0,0.18699200948079428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,64,128,1,fp8,fp8,0,0.22879467407862344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,64,0,1,float16,fp8,0,0.18780799706776938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,64,0,1,fp8,fp8,0,0.22829866409301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,64,128,1,float16,float16,0,0.18742400407791138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,64,0,1,float16,float16,0,0.1872533361117045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,64,128,1,float16,fp8,0,0.18732800086339316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,64,128,1,fp8,fp8,0,0.2299999992052714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,64,0,1,float16,fp8,0,0.1869759956995646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,64,0,1,float16,float16,0,0.19032533963521323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,64,0,1,fp8,fp8,0,0.2304426630338033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,64,128,1,float16,float16,0,0.19025067488352457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,64,128,1,float16,fp8,0,0.1897439956665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,64,0,1,float16,float16,0,0.14541866381963095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,64,128,1,fp8,fp8,0,0.2298133373260498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,64,0,1,float16,fp8,0,0.19035732746124268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,64,0,1,fp8,fp8,0,0.22925333182017008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,64,128,1,float16,float16,0,0.1453013320763906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,64,128,1,float16,fp8,0,0.14037332932154337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,64,128,1,fp8,fp8,0,0.147407998641332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,64,0,1,float16,fp8,0,0.14058132966359457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,64,0,1,fp8,fp8,0,0.1452959974606832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,64,128,1,float16,float16,0,0.09946667154630025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,64,0,1,float16,float16,0,0.1006773312886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,64,128,1,float16,fp8,0,0.09975467125574748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,64,128,1,fp8,fp8,0,0.12618133425712585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,64,0,1,float16,fp8,0,0.10123733679453532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,64,0,1,fp8,fp8,0,0.12661866346995035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,64,128,1,float16,float16,0,0.10108266274134318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,64,0,1,float16,float16,0,0.10103999574979146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,64,128,1,float16,fp8,0,0.10099732875823975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,64,128,1,fp8,fp8,0,0.12708800037701926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,64,0,1,float16,fp8,0,0.10046933094660442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,64,0,1,fp8,fp8,0,0.12634133299191794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,64,128,1,float16,float16,0,0.10152533650398254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,64,0,1,float16,float16,0,0.10106666882832845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,64,128,1,float16,fp8,0,0.1013706624507904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,64,128,1,fp8,fp8,0,0.12755200266838074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,64,0,1,float16,fp8,0,0.10188800096511841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,64,0,1,fp8,fp8,0,0.12662933270136514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,64,128,1,float16,float16,0,0.10275733470916748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,64,0,1,float16,float16,0,0.10310932993888855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,64,128,1,float16,fp8,0,0.10364799698193868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,64,128,1,fp8,fp8,0,0.12811733285586038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,64,0,1,float16,fp8,0,0.10313600301742554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,64,0,1,fp8,fp8,0,0.12866666913032532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,64,128,1,float16,float16,0,0.07874666651089986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,64,0,1,float16,float16,0,0.07895466685295105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,64,0,1,float16,float16,0,0.05620799958705902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,64,128,1,float16,fp8,0,0.07579199969768524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,64,128,1,fp8,fp8,0,0.08298133313655853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,64,0,1,float16,fp8,0,0.07655466596285503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,64,0,1,fp8,fp8,0,0.08401599526405334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,64,128,1,float16,float16,0,0.05608533322811127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,64,128,1,float16,fp8,0,0.05653333167235056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,64,128,1,float16,fp8,0,0.05574933191140493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,64,0,1,float16,fp8,0,0.05600533386071523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,64,128,1,fp8,fp8,0,0.07250133156776428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,64,0,1,fp8,fp8,0,0.07228266696135204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,64,128,1,float16,float16,0,0.056186666091283165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,64,0,1,float16,float16,0,0.05622399846712748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,64,0,1,float16,float16,0,0.0565226674079895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,64,128,1,fp8,fp8,0,0.07152533531188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,64,0,1,float16,fp8,0,0.05676266551017761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,64,0,1,fp8,fp8,0,0.07234666744867961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,64,128,1,float16,float16,0,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,64,128,1,float16,fp8,0,0.05597866574923197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,64,0,1,float16,fp8,0,0.05630399783452352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,64,128,1,fp8,fp8,0,0.0720960001150767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,64,0,1,fp8,fp8,0,0.07293866574764252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,64,128,1,float16,float16,0,0.057029331723848976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,64,0,1,fp8,fp8,0,0.07401599983374278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,64,0,1,float16,float16,0,0.05681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,64,128,1,float16,fp8,0,0.05726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,64,128,1,fp8,fp8,0,0.07355199754238129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,64,0,1,float16,fp8,0,0.05692266424496969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,64,128,1,float16,float16,0,0.03926933308442434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,64,0,1,float16,float16,0,0.039264000952243805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,64,128,1,float16,fp8,0,0.03678400069475174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,64,128,1,fp8,fp8,0,0.048384000857671104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,64,0,1,float16,fp8,0,0.038047999143600464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,64,0,1,fp8,fp8,0,0.048751999934514366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,64,128,1,float16,float16,0,0.031744000812371574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,64,0,1,float16,float16,0,0.03197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,64,128,1,float16,fp8,0,0.03200533241033554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,64,128,1,fp8,fp8,0,0.04162666698296865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,64,0,1,float16,fp8,0,0.03219733387231827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,64,0,1,fp8,fp8,0,0.04168533285458883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,64,128,1,float16,float16,0,0.03243733445803324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,64,0,1,float16,float16,0,0.03206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,64,128,1,float16,float16,0,0.032469332218170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,64,128,1,float16,fp8,0,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,64,128,1,fp8,fp8,0,0.04146133363246918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,64,0,1,float16,fp8,0,0.03225066761175791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,64,0,1,fp8,fp8,0,0.04107200105985006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,64,0,1,float16,float16,0,0.03307733436425527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,64,128,1,float16,fp8,0,0.032826667030652366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,64,0,1,float16,float16,0,0.032613334556420646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,64,128,1,fp8,fp8,0,0.04274133344491323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,64,0,1,float16,fp8,0,0.03241066634654999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,64,0,1,fp8,fp8,0,0.04233066737651825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,64,128,1,float16,float16,0,0.03275733441114426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,64,128,1,float16,fp8,0,0.032986665765444435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,64,128,1,fp8,fp8,0,0.043061330914497375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,64,0,1,float16,fp8,0,0.032672000428040825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,64,0,1,fp8,fp8,0,0.043007999658584595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,64,0,1,float16,fp8,0,0.02460266649723053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,64,128,1,float16,float16,0,0.02421333392461141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,64,0,1,float16,float16,0,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,64,128,1,float16,fp8,0,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,64,128,1,fp8,fp8,0,0.028922667105992634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,64,0,1,fp8,fp8,0,0.028965334097544353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,64,128,1,float16,float16,0,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,64,0,1,fp8,fp8,0,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,64,0,1,float16,float16,0,0.02233600119749705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,64,128,1,float16,fp8,0,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,64,128,1,fp8,fp8,0,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,64,128,1,fp8,fp8,0,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,64,0,1,float16,fp8,0,0.022810667753219604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,64,128,1,float16,float16,0,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,64,0,1,float16,float16,0,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,64,128,1,float16,fp8,0,0.022858666876951855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,64,0,1,float16,fp8,0,0.022592000663280487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,64,0,1,fp8,fp8,0,0.027285332481066387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,64,128,1,float16,float16,0,0.02275199939807256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,64,0,1,float16,float16,0,0.022890667120615642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,64,128,1,float16,fp8,0,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,64,128,1,fp8,fp8,0,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,64,0,1,float16,fp8,0,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,64,0,1,float16,fp8,0,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,64,0,1,fp8,fp8,0,0.0277813325325648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,64,128,1,float16,float16,0,0.022778667509555817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,64,0,1,float16,float16,0,0.02276266614596049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,64,128,1,float16,fp8,0,0.02347733328739802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,64,128,1,fp8,fp8,0,0.02845333268245061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,64,0,1,fp8,fp8,0,0.02869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,64,128,1,float16,float16,0,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,64,0,1,float16,float16,0,0.0184906671444575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,64,128,1,float16,fp8,0,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,64,128,1,fp8,fp8,0,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,64,0,1,float16,fp8,0,0.018645333747069042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,64,0,1,fp8,fp8,0,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,64,128,1,float16,float16,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,64,0,1,float16,float16,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,64,128,1,float16,fp8,0,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,64,128,1,fp8,fp8,0,0.020010666300853092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,64,0,1,float16,fp8,0,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,64,0,1,fp8,fp8,0,0.019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,64,128,1,float16,float16,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,64,0,1,float16,float16,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,64,128,1,float16,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,64,128,1,fp8,fp8,0,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,64,0,1,float16,fp8,0,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,64,0,1,fp8,fp8,0,0.020096000283956528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,64,128,1,float16,float16,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,64,0,1,float16,float16,0,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,64,128,1,float16,fp8,0,0.017946666727463405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,64,128,1,fp8,fp8,0,0.020479999482631683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,64,0,1,float16,fp8,0,0.017701332767804463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,64,0,1,fp8,fp8,0,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,64,128,1,float16,float16,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,64,0,1,float16,float16,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,64,128,1,float16,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,64,128,1,fp8,fp8,0,0.020655999581019085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,64,0,1,float16,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,64,0,1,fp8,fp8,0,0.02054399996995926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,64,128,1,float16,float16,0,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,64,0,1,float16,float16,0,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,64,128,1,float16,fp8,0,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,64,128,1,fp8,fp8,0,0.019482667247454327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,64,0,1,float16,fp8,0,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,64,0,1,fp8,fp8,0,0.019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,64,128,1,float16,float16,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,64,0,1,float16,float16,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,64,128,1,float16,fp8,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,64,128,1,fp8,fp8,0,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,64,0,1,float16,fp8,0,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,64,0,1,fp8,fp8,0,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,64,128,1,float16,float16,0,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,64,0,1,float16,float16,0,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,64,128,1,float16,fp8,0,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,64,128,1,fp8,fp8,0,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,64,0,1,float16,fp8,0,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,64,0,1,fp8,fp8,0,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,64,128,1,float16,float16,0,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,64,0,1,float16,float16,0,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,64,128,1,float16,fp8,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,64,128,1,fp8,fp8,0,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,64,0,1,float16,fp8,0,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,64,0,1,fp8,fp8,0,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,64,128,1,float16,float16,0,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,64,0,1,float16,float16,0,0.0143306665122509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,64,128,1,float16,fp8,0,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,64,128,1,fp8,fp8,0,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,64,0,1,float16,fp8,0,0.01597333326935768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,64,0,1,float16,float16,0,0.32872533798217773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,64,0,1,fp8,fp8,0,0.01959466685851415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,64,128,1,float16,float16,0,0.32870399951934814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,64,128,1,float16,fp8,0,0.32815466324488324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,64,0,1,float16,fp8,0,0.32788266738255817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,64,128,1,fp8,fp8,0,0.5903040170669556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,64,0,1,fp8,fp8,0,0.5878026485443115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,64,128,1,float16,float16,0,0.33169599374135333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,64,0,1,float16,float16,0,0.3325706720352173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,64,128,1,float16,fp8,0,0.33168532450993854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,64,0,1,float16,fp8,0,0.3311946590741475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,64,128,1,fp8,fp8,0,0.5882613261540731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,64,128,1,float16,float16,0,0.3349066575368245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,64,0,1,float16,float16,0,0.3358773390452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,64,0,1,fp8,fp8,0,0.591759999593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,64,128,1,float16,fp8,0,0.3361440102259318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,64,128,1,fp8,fp8,0,0.5873973369598389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,64,0,1,float16,fp8,0,0.33508265018463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,64,0,1,fp8,fp8,0,0.5871626536051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,64,0,1,float16,float16,0,0.341754674911499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,64,128,1,float16,float16,0,0.3405760129292806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,64,128,1,float16,fp8,0,0.33933866024017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,64,0,1,float16,fp8,0,0.33816532293955487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,64,128,1,fp8,fp8,0,0.5912799835205078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,64,128,1,float16,float16,0,0.27003200848897296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,64,0,1,fp8,fp8,0,0.590336004892985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,64,0,1,float16,float16,0,0.26968000332514447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,64,0,1,float16,fp8,0,0.25854400793711346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,64,128,1,float16,fp8,0,0.2595626711845398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,64,128,1,fp8,fp8,0,0.34384000301361084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,64,128,1,float16,float16,0,0.17272533973058066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,64,0,1,fp8,fp8,0,0.34377066294352215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,64,0,1,float16,fp8,0,0.17252800861994425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,64,0,1,float16,float16,0,0.1723360021909078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,64,128,1,float16,fp8,0,0.17354132731755575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,64,128,1,fp8,fp8,0,0.3051946759223938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,64,0,1,fp8,fp8,0,0.3046293258666992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,64,128,1,float16,float16,0,0.17383466164271036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,64,0,1,float16,float16,0,0.1736746629079183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,64,128,1,float16,fp8,0,0.17401599884033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,64,128,1,fp8,fp8,0,0.30483200152715045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,64,0,1,float16,fp8,0,0.17327467600504556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,64,0,1,fp8,fp8,0,0.30447999636332196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,64,128,1,float16,float16,0,0.17491199572881064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,64,0,1,float16,fp8,0,0.17454934120178223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,64,0,1,float16,float16,0,0.17495467265446982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,64,128,1,float16,fp8,0,0.17502933740615845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,64,128,1,fp8,fp8,0,0.3046826720237732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,64,0,1,fp8,fp8,0,0.30311999718348187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,64,128,1,float16,float16,0,0.17650665839513144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,64,0,1,float16,float16,0,0.17717333634694418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,64,128,1,float16,fp8,0,0.1764799952507019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,64,0,1,float16,fp8,0,0.1774453322092692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,64,128,1,fp8,fp8,0,0.307589332262675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,64,128,1,float16,float16,0,0.14202133814493814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,64,128,1,fp8,fp8,0,0.18237866957982382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,64,0,1,fp8,fp8,0,0.30664533376693726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,64,0,1,float16,float16,0,0.14150399963061014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,64,128,1,float16,fp8,0,0.1368106702963511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,64,0,1,float16,fp8,0,0.13643733660380045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,64,0,1,fp8,fp8,0,0.18292266130447388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,64,128,1,float16,float16,0,0.09406399726867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,64,0,1,float16,float16,0,0.09400000174840291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,64,128,1,float16,fp8,0,0.09434133768081665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,64,128,1,fp8,fp8,0,0.1656000018119812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,64,0,1,float16,fp8,0,0.09368000427881877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,64,0,1,fp8,fp8,0,0.16571199893951416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,64,128,1,float16,float16,0,0.09436266620953877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,64,0,1,float16,float16,0,0.09511466821034749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,64,128,1,float16,fp8,0,0.0957973301410675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,64,128,1,fp8,fp8,0,0.16427200039227804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,64,0,1,float16,fp8,0,0.09539733330408733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,64,0,1,fp8,fp8,0,0.16480533281962076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,64,128,1,float16,float16,0,0.09469333291053772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,64,0,1,float16,float16,0,0.09551466504732768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,64,128,1,float16,fp8,0,0.0956053336461385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,64,128,1,fp8,fp8,0,0.1646933356920878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,64,0,1,float16,fp8,0,0.09569066762924194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,64,0,1,fp8,fp8,0,0.16536532839139303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,64,128,1,float16,float16,0,0.09708266456921895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,64,128,1,float16,float16,0,0.07494933406511943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,64,0,1,fp8,fp8,0,0.16609600186347961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,64,0,1,float16,float16,0,0.09722666939099629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,64,128,1,float16,fp8,0,0.09697600205739339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,64,128,1,fp8,fp8,0,0.16761600971221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,64,0,1,float16,fp8,0,0.09714133540789287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,64,0,1,float16,float16,0,0.07542400062084198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,64,128,1,float16,fp8,0,0.07288533449172974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,64,128,1,fp8,fp8,0,0.10078932841618855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,64,0,1,float16,fp8,0,0.07268266876538594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,64,0,1,fp8,fp8,0,0.10032000144322713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,64,128,1,float16,float16,0,0.05259733398755392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,64,0,1,float16,float16,0,0.05305066704750061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,64,128,1,float16,fp8,0,0.05340800185998281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,64,128,1,fp8,fp8,0,0.09274133046468098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,64,0,1,float16,fp8,0,0.05362133185068766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,64,0,1,fp8,fp8,0,0.09126933415730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,64,128,1,float16,float16,0,0.052933335304260254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,64,0,1,float16,float16,0,0.05258133510748545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,64,128,1,float16,fp8,0,0.053264002005259194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,64,128,1,fp8,fp8,0,0.09119466940561931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,64,0,1,float16,fp8,0,0.053210665782292686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,64,0,1,fp8,fp8,0,0.09199466307957967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,64,128,1,float16,float16,0,0.052986666560173035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,64,0,1,float16,float16,0,0.05324266850948334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,64,128,1,float16,fp8,0,0.05342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,64,128,1,fp8,fp8,0,0.09141332904497783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,64,0,1,float16,fp8,0,0.05392000079154968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,64,0,1,fp8,fp8,0,0.09099200367927551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,64,0,1,float16,fp8,0,0.054010664423306785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,64,128,1,float16,float16,0,0.053904001911481224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,64,0,1,float16,float16,0,0.05379733443260193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,64,128,1,float16,fp8,0,0.054287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,64,128,1,fp8,fp8,0,0.09197866916656494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,64,128,1,fp8,fp8,0,0.058490668733914696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,64,0,1,fp8,fp8,0,0.09245333075523376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,64,128,1,float16,float16,0,0.03775999943415324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,64,0,1,float16,float16,0,0.03929600119590759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,64,128,1,float16,fp8,0,0.0371573343873024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,64,0,1,float16,fp8,0,0.03673599908749262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,64,0,1,fp8,fp8,0,0.058042665322621666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,64,128,1,float16,float16,0,0.031162666777769726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,64,0,1,float16,float16,0,0.030933332939942677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,64,128,1,float16,fp8,0,0.03126933425664902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,64,128,1,fp8,fp8,0,0.052186667919158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,64,0,1,float16,fp8,0,0.03150933235883713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,64,0,1,fp8,fp8,0,0.05234666665395101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,64,0,1,float16,fp8,0,0.03125333289305369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,64,0,1,fp8,fp8,0,0.05273066461086273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,64,128,1,float16,float16,0,0.031231999397277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,64,0,1,float16,float16,0,0.03212266663710276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,64,0,1,float16,float16,0,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,64,128,1,float16,fp8,0,0.031343999008337654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,64,128,1,fp8,fp8,0,0.05247466762860616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,64,128,1,float16,float16,0,0.03223466624816259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,64,128,1,float16,fp8,0,0.03252800057331721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,64,128,1,fp8,fp8,0,0.052005335688591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,64,0,1,float16,fp8,0,0.03213333338499069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,64,0,1,fp8,fp8,0,0.052746668457984924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,64,128,1,float16,float16,0,0.03212266663710276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,64,0,1,float16,float16,0,0.032229334115982056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,64,128,1,float16,fp8,0,0.03238933285077413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,64,128,1,fp8,fp8,0,0.05348266661167145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,64,0,1,float16,fp8,0,0.032138665517171226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,64,0,1,fp8,fp8,0,0.053354665637016296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,64,0,1,fp8,fp8,0,0.03421333432197571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,64,128,1,float16,float16,0,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,64,0,1,float16,float16,0,0.021967999637126923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,64,128,1,float16,fp8,0,0.022111999491850536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,64,128,1,fp8,fp8,0,0.03499199946721395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,64,0,1,float16,fp8,0,0.022890667120615642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,64,128,1,float16,float16,0,0.020341333001852036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,64,0,1,float16,float16,0,0.02025066688656807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,64,128,1,float16,fp8,0,0.020794666061798733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,64,128,1,fp8,fp8,0,0.03311999887228012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,64,0,1,float16,fp8,0,0.02041600023706754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,64,0,1,fp8,fp8,0,0.03297599901755651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,64,128,1,float16,float16,0,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,64,0,1,float16,float16,0,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,64,128,1,float16,fp8,0,0.02086399992307027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,64,128,1,fp8,fp8,0,0.032746667663256325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,64,0,1,float16,fp8,0,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,64,0,1,fp8,fp8,0,0.03293866664171219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,64,128,1,float16,float16,0,0.02090666691462199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,64,0,1,float16,float16,0,0.020703999946514767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,64,128,1,float16,fp8,0,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,64,128,1,fp8,fp8,0,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,64,0,1,float16,fp8,0,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,64,0,1,fp8,fp8,0,0.03309866786003113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,64,128,1,float16,float16,0,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,64,0,1,float16,float16,0,0.02088533341884613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,64,128,1,float16,fp8,0,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,64,128,1,fp8,fp8,0,0.033285332222779594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,64,0,1,float16,fp8,0,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,64,0,1,fp8,fp8,0,0.03349866718053818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,64,128,1,float16,float16,0,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,64,0,1,float16,float16,0,0.016762666404247284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,64,128,1,float16,fp8,0,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,64,128,1,fp8,fp8,0,0.024357333779335022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,64,0,1,float16,fp8,0,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,64,0,1,fp8,fp8,0,0.02422933280467987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,64,128,1,float16,float16,0,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,64,0,1,float16,float16,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,64,128,1,float16,fp8,0,0.016352000335852306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,64,128,1,fp8,fp8,0,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,64,0,1,float16,fp8,0,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,64,0,1,fp8,fp8,0,0.022890667120615642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,64,128,1,float16,float16,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,64,0,1,float16,float16,0,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,64,128,1,float16,fp8,0,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,64,128,1,fp8,fp8,0,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,64,0,1,float16,fp8,0,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,64,0,1,fp8,fp8,0,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,64,128,1,float16,float16,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,64,0,1,float16,float16,0,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,64,128,1,float16,fp8,0,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,64,128,1,fp8,fp8,0,0.023984000086784363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,64,0,1,float16,fp8,0,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,64,0,1,fp8,fp8,0,0.024138666689395905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,64,128,1,float16,float16,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,64,0,1,float16,float16,0,0.016741332908471424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,64,128,1,float16,fp8,0,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,64,128,1,fp8,fp8,0,0.023978665471076965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,64,0,1,float16,fp8,0,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,64,0,1,fp8,fp8,0,0.02404266595840454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,64,128,1,float16,float16,0,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,64,0,1,float16,float16,0,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,64,128,1,float16,fp8,0,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,64,128,1,fp8,fp8,0,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,64,0,1,float16,fp8,0,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,64,0,1,fp8,fp8,0,0.019802667200565338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,64,0,1,float16,fp8,0,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,64,128,1,float16,float16,0,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,64,0,1,float16,float16,0,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,64,128,1,float16,fp8,0,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,64,128,1,fp8,fp8,0,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,64,0,1,fp8,fp8,0,0.018768000106016796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,64,128,1,float16,float16,0,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,64,0,1,float16,float16,0,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,64,128,1,float16,fp8,0,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,64,128,1,fp8,fp8,0,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,64,0,1,float16,fp8,0,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,64,0,1,fp8,fp8,0,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,64,128,1,float16,float16,0,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,64,128,1,float16,float16,0,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,64,0,1,float16,float16,0,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,64,128,1,float16,fp8,0,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,64,128,1,fp8,fp8,0,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,64,0,1,float16,fp8,0,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,64,0,1,fp8,fp8,0,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,64,0,1,float16,float16,0,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,64,128,1,float16,fp8,0,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,64,128,1,fp8,fp8,0,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,64,0,1,float16,fp8,0,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,64,0,1,fp8,fp8,0,0.019573333362738293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,64,128,1,float16,float16,0,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,64,0,1,float16,float16,0,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,64,128,1,float16,fp8,0,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,64,128,1,fp8,fp8,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,64,0,1,float16,fp8,0,0.012335999558369318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,64,0,1,fp8,fp8,0,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,64,128,1,float16,float16,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,64,0,1,float16,float16,0,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,64,128,1,float16,fp8,0,0.012538666526476542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,64,128,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,64,0,1,float16,fp8,0,0.012378666549921036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,64,0,1,fp8,fp8,0,0.01773333301146825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,64,128,1,float16,float16,0,0.012170666207869848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,64,0,1,float16,float16,0,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,64,128,1,float16,fp8,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,64,128,1,fp8,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,64,0,1,float16,fp8,0,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,64,0,1,fp8,fp8,0,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,64,128,1,float16,float16,0,0.012074666718641916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,64,0,1,float16,float16,0,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,64,128,1,float16,fp8,0,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,64,128,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,64,0,1,float16,fp8,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,64,0,1,fp8,fp8,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,64,128,1,float16,float16,0,0.012234666695197424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,64,0,1,float16,float16,0,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,64,128,1,float16,fp8,0,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,64,128,1,fp8,fp8,0,0.018330667167901993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,64,0,1,float16,fp8,0,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,64,0,1,fp8,fp8,0,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,64,128,1,float16,float16,0,0.19595199823379517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,64,0,1,float16,float16,0,0.1967359979947408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,64,128,1,float16,fp8,0,0.1957226594289144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,64,0,1,float16,fp8,0,0.19697066148122153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,64,128,1,fp8,fp8,0,0.48791468143463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,64,128,1,float16,float16,0,0.1965493361155192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,64,0,1,float16,float16,0,0.19703465700149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,64,0,1,fp8,fp8,0,0.48768532276153564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,64,128,1,float16,fp8,0,0.1965706745783488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,64,0,1,float16,fp8,0,0.19749865929285684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,64,128,1,fp8,fp8,0,0.4896373351414998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,64,128,1,float16,float16,0,0.19857599337895712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,64,0,1,fp8,fp8,0,0.4886453151702881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,64,0,1,float16,float16,0,0.1995840072631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,64,128,1,float16,fp8,0,0.1981119910875956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,64,0,1,float16,fp8,0,0.1995840072631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,64,128,1,fp8,fp8,0,0.4877920150756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,64,128,1,float16,float16,0,0.2036959926287333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,64,0,1,float16,float16,0,0.2029119928677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,64,0,1,fp8,fp8,0,0.48768532276153564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,64,128,1,float16,fp8,0,0.20301334063212076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,64,128,1,float16,float16,0,0.14168000221252441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,64,0,1,fp8,fp8,0,0.48974935213724774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,64,0,1,float16,fp8,0,0.2026453415552775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,64,128,1,fp8,fp8,0,0.2759466568628947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,64,128,1,fp8,fp8,0,0.48900798956553143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,64,0,1,float16,float16,0,0.14192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,64,128,1,float16,fp8,0,0.1372106671333313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,64,0,1,float16,fp8,0,0.13705066839853922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,64,0,1,fp8,fp8,0,0.2757280071576436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,64,128,1,float16,float16,0,0.10378666718800862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,64,0,1,float16,float16,0,0.10397866368293762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,64,128,1,float16,fp8,0,0.10434666275978088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,64,0,1,float16,fp8,0,0.10408000151316325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,64,128,1,fp8,fp8,0,0.2582506736119588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,64,0,1,fp8,fp8,0,0.2569013237953186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,64,128,1,float16,float16,0,0.1046453317006429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,64,0,1,float16,float16,0,0.10506133238474528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,64,128,1,float16,fp8,0,0.10546132922172546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,64,128,1,fp8,fp8,0,0.25729066133499146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,64,0,1,float16,fp8,0,0.10522133111953735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,64,0,1,fp8,fp8,0,0.2577333251635234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,64,128,1,float16,float16,0,0.10572800040245056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,64,0,1,float16,float16,0,0.10445866982142131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,64,128,1,float16,fp8,0,0.10517332951227824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,64,128,1,fp8,fp8,0,0.25682665904362995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,64,0,1,float16,fp8,0,0.10585600137710571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,64,128,1,float16,float16,0,0.10801066954930623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,64,0,1,fp8,fp8,0,0.2566080093383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,64,0,1,float16,float16,0,0.1081813375155131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,64,128,1,float16,fp8,0,0.10744532942771912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,64,0,1,float16,fp8,0,0.10734400153160095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,64,128,1,fp8,fp8,0,0.25870933135350543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,64,128,1,float16,float16,0,0.07526400188604991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,64,0,1,fp8,fp8,0,0.258245329062144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,64,0,1,float16,float16,0,0.07544533411661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,64,128,1,float16,fp8,0,0.07284266750017802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,64,128,1,fp8,fp8,0,0.14777599771817526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,64,0,1,float16,fp8,0,0.07339733342329662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,64,0,1,fp8,fp8,0,0.15001599987347922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,64,128,1,float16,float16,0,0.05694933235645294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,64,0,1,float16,float16,0,0.05717866619427999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,64,128,1,float16,fp8,0,0.05658666789531708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,64,128,1,fp8,fp8,0,0.13850133617719015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,64,0,1,float16,fp8,0,0.056799997886021934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,64,0,1,fp8,fp8,0,0.13797866304715475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,64,128,1,float16,float16,0,0.05682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,64,0,1,float16,float16,0,0.05701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,64,128,1,float16,fp8,0,0.05696000158786774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,64,128,1,fp8,fp8,0,0.1383680005868276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,64,0,1,float16,fp8,0,0.057562669118245445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,64,0,1,float16,fp8,0,0.05736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,64,0,1,fp8,fp8,0,0.13834666212399802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,64,128,1,float16,float16,0,0.0572266678015391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,64,128,1,float16,fp8,0,0.057317331433296204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,64,0,1,float16,float16,0,0.0574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,64,128,1,fp8,fp8,0,0.13824533422787985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,64,0,1,fp8,fp8,0,0.13801599542299905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,64,128,1,float16,float16,0,0.05851200222969055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,64,0,1,float16,float16,0,0.05795733133951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,64,128,1,float16,fp8,0,0.058304001887639366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,64,128,1,fp8,fp8,0,0.13948800166447958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,64,0,1,float16,fp8,0,0.05831466615200043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,64,0,1,fp8,fp8,0,0.13833600282669067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,64,128,1,float16,float16,0,0.03794133414824804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,64,0,1,float16,float16,0,0.03738133360942205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,64,128,1,float16,fp8,0,0.03573333223660787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,64,128,1,float16,fp8,0,0.03283733377854029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,64,128,1,fp8,fp8,0,0.08179733157157898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,64,0,1,float16,fp8,0,0.03656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,64,0,1,fp8,fp8,0,0.08163733283678691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,64,128,1,float16,float16,0,0.03259200106064478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,64,0,1,float16,float16,0,0.03256533294916153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,64,128,1,fp8,fp8,0,0.07521600027879079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,64,0,1,float16,fp8,0,0.032287999987602234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,64,0,1,fp8,fp8,0,0.0773226668437322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,64,128,1,float16,float16,0,0.03286933402220408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,64,0,1,float16,float16,0,0.03270933280388514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,64,128,1,float16,fp8,0,0.032431999842325844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,64,128,1,float16,fp8,0,0.03389333436886469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,64,128,1,fp8,fp8,0,0.07560533285140991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,64,0,1,float16,fp8,0,0.03276800115903219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,64,0,1,fp8,fp8,0,0.07655466596285503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,64,128,1,float16,float16,0,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,64,0,1,float16,float16,0,0.033557333052158356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,64,128,1,fp8,fp8,0,0.07704000174999237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,64,0,1,float16,fp8,0,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,64,0,1,fp8,fp8,0,0.07704000174999237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,64,128,1,float16,float16,0,0.03332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,64,0,1,float16,float16,0,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,64,128,1,float16,fp8,0,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,64,128,1,fp8,fp8,0,0.07674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,64,0,1,float16,fp8,0,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,64,0,1,fp8,fp8,0,0.07690133154392242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,64,128,1,float16,float16,0,0.02276266614596049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,64,0,1,float16,float16,0,0.02269333352645238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,64,128,1,float16,fp8,0,0.022085333863894146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,64,128,1,fp8,fp8,0,0.04728533327579498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,64,0,1,float16,fp8,0,0.02275199939807256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,64,0,1,fp8,fp8,0,0.0470773329337438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,64,128,1,float16,float16,0,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,64,0,1,float16,float16,0,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,64,128,1,float16,fp8,0,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,64,128,1,fp8,fp8,0,0.04554666578769684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,64,0,1,float16,fp8,0,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,64,0,1,fp8,fp8,0,0.04613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,64,128,1,float16,float16,0,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,64,0,1,float16,float16,0,0.021568000316619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,64,128,1,float16,fp8,0,0.021914665897687275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,64,128,1,fp8,fp8,0,0.04587733248869578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,64,0,1,float16,fp8,0,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,64,0,1,fp8,fp8,0,0.04573333263397217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,64,128,1,float16,float16,0,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,64,0,1,float16,float16,0,0.02181333303451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,64,128,1,float16,fp8,0,0.022074667116006214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,64,128,1,fp8,fp8,0,0.046853333711624146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,64,0,1,float16,fp8,0,0.02184533327817917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,64,0,1,fp8,fp8,0,0.045781334241231285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,64,128,1,float16,float16,0,0.02183466653029124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,64,0,1,float16,float16,0,0.021685334543387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,64,128,1,float16,fp8,0,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,64,128,1,fp8,fp8,0,0.046538665890693665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,64,0,1,float16,fp8,0,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,64,0,1,fp8,fp8,0,0.045738667249679565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,64,128,1,float16,float16,0,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,64,0,1,float16,float16,0,0.016165333489576977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,64,128,1,float16,float16,0,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,64,0,1,float16,float16,0,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,64,128,1,float16,fp8,0,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,64,128,1,fp8,fp8,0,0.031162666777769726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,64,0,1,float16,fp8,0,0.016352000335852306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,64,0,1,fp8,fp8,0,0.030661332110563915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,64,128,1,float16,fp8,0,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,64,128,1,fp8,fp8,0,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,64,0,1,float16,fp8,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,64,0,1,fp8,fp8,0,0.03017599880695343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,64,128,1,float16,float16,0,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,64,128,1,float16,float16,0,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,64,0,1,float16,float16,0,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,64,128,1,float16,fp8,0,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,64,128,1,fp8,fp8,0,0.029802667597929638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,64,0,1,float16,fp8,0,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,64,0,1,fp8,fp8,0,0.03013866643110911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,64,0,1,float16,float16,0,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,64,128,1,float16,fp8,0,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,64,128,1,fp8,fp8,0,0.030410667260487873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,64,0,1,float16,fp8,0,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,64,0,1,fp8,fp8,0,0.029882666965325672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,64,0,1,fp8,fp8,0,0.030192000170548756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,64,128,1,float16,float16,0,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,64,0,1,float16,float16,0,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,64,128,1,float16,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,64,128,1,fp8,fp8,0,0.02980799973011017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,64,0,1,float16,fp8,0,0.015583999454975128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,64,128,1,float16,float16,0,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,64,0,1,float16,float16,0,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,64,128,1,float16,fp8,0,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,64,128,1,fp8,fp8,0,0.02181866765022278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,64,128,1,fp8,fp8,0,0.022618666291236877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,64,0,1,float16,fp8,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,64,0,1,float16,fp8,0,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,64,0,1,fp8,fp8,0,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,64,128,1,float16,float16,0,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,64,0,1,float16,float16,0,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,64,128,1,float16,fp8,0,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,64,0,1,fp8,fp8,0,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,64,128,1,float16,float16,0,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,64,0,1,float16,float16,0,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,64,128,1,float16,fp8,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,64,128,1,fp8,fp8,0,0.022917332748572033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,64,0,1,float16,fp8,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,64,0,1,fp8,fp8,0,0.022453332940737408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,64,128,1,float16,float16,0,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,64,0,1,float16,float16,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,64,128,1,float16,fp8,0,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,64,128,1,fp8,fp8,0,0.022416000564893086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,64,128,1,fp8,fp8,0,0.02254933367172877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,64,0,1,float16,fp8,0,0.013482666263977686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,64,0,1,fp8,fp8,0,0.02229333420594533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,64,128,1,float16,float16,0,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,64,0,1,float16,float16,0,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,64,128,1,float16,fp8,0,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,64,0,1,float16,fp8,0,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,64,0,1,fp8,fp8,0,0.022357332209746044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,64,128,1,float16,float16,0,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,64,0,1,float16,float16,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,64,128,1,float16,fp8,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,64,128,1,fp8,fp8,0,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,64,0,1,float16,fp8,0,0.012175999581813812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,64,0,1,fp8,fp8,0,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,64,128,1,float16,float16,0,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,64,0,1,float16,float16,0,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,64,128,1,float16,fp8,0,0.012170666207869848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,64,128,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,64,0,1,float16,fp8,0,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,64,0,1,fp8,fp8,0,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,64,128,1,float16,float16,0,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,64,0,1,float16,float16,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,64,128,1,float16,fp8,0,0.012159999459981918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,64,128,1,fp8,fp8,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,64,0,1,float16,fp8,0,0.012223999947309494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,64,0,1,fp8,fp8,0,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,64,128,1,float16,float16,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,64,0,1,float16,float16,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,64,128,1,float16,fp8,0,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,64,128,1,fp8,fp8,0,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,64,0,1,float16,fp8,0,0.01221866657336553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,64,0,1,fp8,fp8,0,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,64,0,1,fp8,fp8,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,64,128,1,float16,float16,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,64,0,1,float16,float16,0,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,64,128,1,float16,fp8,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,64,128,1,float16,fp8,0,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,64,128,1,fp8,fp8,0,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,64,0,1,float16,fp8,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,64,128,1,float16,float16,0,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,64,0,1,float16,float16,0,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,64,0,1,float16,float16,0,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,64,128,1,fp8,fp8,0,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,64,0,1,float16,fp8,0,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,64,0,1,fp8,fp8,0,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,64,128,1,float16,float16,0,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,64,128,1,float16,fp8,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,64,128,1,fp8,fp8,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,64,0,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,64,0,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,64,128,1,float16,float16,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,64,0,1,float16,float16,0,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,64,128,1,float16,fp8,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,64,128,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,64,0,1,fp8,fp8,0,0.018058666338523228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,64,128,1,float16,float16,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,64,0,1,float16,float16,0,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,64,128,1,float16,fp8,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,64,128,1,fp8,fp8,0,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,64,0,1,float16,fp8,0,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,64,0,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,64,128,1,float16,float16,0,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,64,0,1,float16,float16,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,64,128,1,float16,fp8,0,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,64,128,1,fp8,fp8,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,64,0,1,float16,fp8,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,64,0,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,64,128,1,float16,float16,0,0.14266666769981384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,64,0,1,float16,fp8,0,0.14307733376820883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,64,0,1,float16,float16,0,0.14262400070826212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,64,128,1,float16,fp8,0,0.14338666200637817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,64,128,1,fp8,fp8,0,0.44070935249328613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,64,128,1,float16,float16,0,0.1432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,64,0,1,float16,float16,0,0.14292266964912415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,64,0,1,fp8,fp8,0,0.4407680034637451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,64,128,1,float16,fp8,0,0.14326399564743042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,64,128,1,fp8,fp8,0,0.441866676012675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,64,0,1,float16,fp8,0,0.14324800173441568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,64,128,1,float16,float16,0,0.1442080040772756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,64,0,1,fp8,fp8,0,0.44363200664520264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,64,0,1,float16,float16,0,0.14446933070818582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,64,128,1,float16,fp8,0,0.14390933513641357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,64,0,1,float16,fp8,0,0.1437226633230845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,64,128,1,fp8,fp8,0,0.4405333201090495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,64,128,1,float16,float16,0,0.14516266187032065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,64,0,1,float16,float16,0,0.14479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,64,0,1,fp8,fp8,0,0.4424639940261841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,64,128,1,float16,fp8,0,0.14475199580192566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,64,0,1,float16,fp8,0,0.144378662109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,64,128,1,fp8,fp8,0,0.442415992418925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,64,128,1,float16,float16,0,0.08880533774693807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,64,0,1,float16,float16,0,0.08920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,64,0,1,fp8,fp8,0,0.4426453510920207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,64,128,1,float16,fp8,0,0.08623466889063518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,64,128,1,fp8,fp8,0,0.23990400632222494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,64,0,1,float16,fp8,0,0.08719467123349507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,64,128,1,float16,float16,0,0.07633066674073537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,64,0,1,fp8,fp8,0,0.2400053342183431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,64,0,1,float16,float16,0,0.07641066610813141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,64,128,1,float16,fp8,0,0.07653866708278656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,64,128,1,fp8,fp8,0,0.2299733360608419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,64,0,1,float16,fp8,0,0.07578666508197784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,64,0,1,fp8,fp8,0,0.2295786738395691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,64,128,1,float16,float16,0,0.07626666625340779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,64,0,1,float16,float16,0,0.07590400179227193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,64,128,1,float16,fp8,0,0.07668266693751018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,64,128,1,fp8,fp8,0,0.23026132583618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,64,0,1,float16,fp8,0,0.07679466903209686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,64,128,1,float16,float16,0,0.07700266440709432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,64,0,1,fp8,fp8,0,0.2299626668294271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,64,0,1,float16,float16,0,0.07656533519426982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,64,128,1,float16,fp8,0,0.07704533139864604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,64,128,1,fp8,fp8,0,0.2311733365058899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,64,128,1,float16,fp8,0,0.07761600116888683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,64,0,1,float16,fp8,0,0.07706133524576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,64,0,1,fp8,fp8,0,0.22992533445358276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,64,128,1,float16,float16,0,0.077674667040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,64,0,1,float16,float16,0,0.07765866816043854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,64,0,1,float16,fp8,0,0.07791999975840251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,64,128,1,fp8,fp8,0,0.2312906583150228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,64,0,1,fp8,fp8,0,0.23107733329137167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,64,128,1,float16,float16,0,0.045007998744646706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,64,0,1,float16,float16,0,0.04550399879614512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,64,128,1,float16,fp8,0,0.04450666904449463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,64,128,1,fp8,fp8,0,0.1292693316936493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,64,0,1,float16,fp8,0,0.04444799820582072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,64,0,1,fp8,fp8,0,0.12935466567675272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,64,128,1,float16,float16,0,0.041946664452552795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,64,0,1,float16,float16,0,0.041984001795450844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,64,128,1,float16,fp8,0,0.04215466479460398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,64,128,1,fp8,fp8,0,0.12340266505877177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,64,0,1,float16,fp8,0,0.04201066493988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,64,0,1,fp8,fp8,0,0.12362666924794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,64,128,1,float16,float16,0,0.04252266883850098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,64,0,1,float16,float16,0,0.042170668641726174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,64,128,1,float16,fp8,0,0.04221866528193156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,64,128,1,fp8,fp8,0,0.12239467104276021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,64,0,1,float16,fp8,0,0.04230933388074239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,64,0,1,fp8,fp8,0,0.12221866846084595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,64,128,1,float16,float16,0,0.04329599936803182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,64,0,1,float16,float16,0,0.04348266621430715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,64,128,1,float16,fp8,0,0.04326933125654856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,64,128,1,fp8,fp8,0,0.12329066793123881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,64,0,1,float16,fp8,0,0.04338666796684265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,64,0,1,fp8,fp8,0,0.12387200196584065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,64,128,1,float16,float16,0,0.042992000778516136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,64,0,1,float16,float16,0,0.04312000175317129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,64,128,1,float16,fp8,0,0.04362666606903076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,64,128,1,fp8,fp8,0,0.12452266613642375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,64,0,1,float16,fp8,0,0.04354133208592733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,64,0,1,fp8,fp8,0,0.12426132957140605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,64,128,1,float16,float16,0,0.02643200010061264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,64,0,1,float16,float16,0,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,64,128,1,float16,fp8,0,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,64,128,1,fp8,fp8,0,0.07108800113201141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,64,0,1,float16,fp8,0,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,64,0,1,fp8,fp8,0,0.07056533296902974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,64,128,1,float16,float16,0,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,64,0,1,float16,float16,0,0.025957333544890087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,64,128,1,float16,fp8,0,0.02628266563018163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,64,128,1,fp8,fp8,0,0.06994133194287618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,64,0,1,float16,fp8,0,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,64,0,1,fp8,fp8,0,0.0691893349091212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,64,128,1,float16,float16,0,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,64,0,1,float16,float16,0,0.026352000733216602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,64,128,1,float16,fp8,0,0.02625600000222524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,64,128,1,fp8,fp8,0,0.06905599931875865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,64,0,1,float16,fp8,0,0.02641066660483678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,64,128,1,fp8,fp8,0,0.07008533179759979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,64,0,1,fp8,fp8,0,0.06904533505439758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,64,128,1,float16,float16,0,0.026357332865397137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,64,0,1,float16,float16,0,0.026378666361172993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,64,128,1,float16,fp8,0,0.026506667335828144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,64,0,1,float16,fp8,0,0.02679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,64,0,1,fp8,fp8,0,0.0706826647122701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,64,128,1,float16,float16,0,0.026698666314284008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,64,0,1,float16,float16,0,0.026373334228992462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,64,128,1,float16,fp8,0,0.02661866694688797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,64,128,1,fp8,fp8,0,0.06991466879844666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,64,0,1,float16,fp8,0,0.026613332331180573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,64,0,1,fp8,fp8,0,0.06937600175539653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,64,128,1,float16,float16,0,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,64,0,1,float16,float16,0,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,64,128,1,float16,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,64,128,1,fp8,fp8,0,0.04327466587225596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,64,0,1,float16,fp8,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,64,0,1,fp8,fp8,0,0.04288533329963684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,64,128,1,float16,float16,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,64,0,1,float16,float16,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,64,128,1,float16,fp8,0,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,64,128,1,fp8,fp8,0,0.04190933207670847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,64,0,1,float16,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,64,0,1,fp8,fp8,0,0.04179200033346812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,64,128,1,float16,float16,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,64,0,1,float16,float16,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,64,128,1,float16,fp8,0,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,64,128,1,fp8,fp8,0,0.042080000042915344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,64,0,1,float16,fp8,0,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,64,0,1,fp8,fp8,0,0.04193066557248434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,64,128,1,float16,float16,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,64,128,1,float16,float16,0,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,64,0,1,float16,float16,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,64,128,1,float16,fp8,0,0.018330667167901993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,64,128,1,fp8,fp8,0,0.04276266694068909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,64,0,1,float16,fp8,0,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,64,0,1,fp8,fp8,0,0.04244266450405121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,64,0,1,float16,float16,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,64,128,1,float16,fp8,0,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,64,128,1,fp8,fp8,0,0.04274666806062063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,64,0,1,float16,fp8,0,0.0184906671444575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,64,0,1,fp8,fp8,0,0.043023998538653054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,64,128,1,float16,float16,0,0.014042666802803675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,64,0,1,float16,float16,0,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,64,128,1,float16,fp8,0,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,64,128,1,fp8,fp8,0,0.028570666909217834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,64,0,1,float16,fp8,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,64,0,1,fp8,fp8,0,0.028991999725500744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,64,128,1,float16,float16,0,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,64,0,1,float16,float16,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,64,128,1,float16,fp8,0,0.014042666802803675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,64,128,1,fp8,fp8,0,0.028175999720891316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,64,0,1,float16,fp8,0,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,64,0,1,fp8,fp8,0,0.02808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,64,0,1,fp8,fp8,0,0.028416000306606293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,64,128,1,float16,float16,0,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,64,0,1,float16,float16,0,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,64,128,1,float16,fp8,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,64,128,1,fp8,fp8,0,0.028160000840822857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,64,0,1,float16,fp8,0,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,64,128,1,float16,float16,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,64,0,1,float16,float16,0,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,64,128,1,float16,fp8,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,64,128,1,fp8,fp8,0,0.028751999139785767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,64,0,1,float16,fp8,0,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,64,128,1,fp8,fp8,0,0.029098667204380035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,64,0,1,fp8,fp8,0,0.028255999088287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,64,128,1,float16,float16,0,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,64,0,1,float16,float16,0,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,64,128,1,float16,fp8,0,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,64,0,1,float16,fp8,0,0.014287999520699183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,64,0,1,fp8,fp8,0,0.028405333558718365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,64,128,1,float16,float16,0,0.01227733368674914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,64,0,1,float16,float16,0,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,64,128,1,float16,fp8,0,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,64,128,1,fp8,fp8,0,0.022319999833901722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,64,0,1,float16,fp8,0,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,64,0,1,fp8,fp8,0,0.022053333620230358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,64,128,1,float16,float16,0,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,64,0,1,float16,float16,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,64,128,1,float16,fp8,0,0.012170666207869848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,64,128,1,fp8,fp8,0,0.02146666745344798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,64,0,1,float16,fp8,0,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,64,0,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,64,128,1,float16,float16,0,0.012240000069141388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,64,128,1,float16,float16,0,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,64,0,1,float16,float16,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,64,128,1,float16,fp8,0,0.01228800043463707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,64,128,1,fp8,fp8,0,0.021967999637126923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,64,0,1,float16,fp8,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,64,0,1,fp8,fp8,0,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,64,0,1,float16,float16,0,0.012346666306257248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,64,128,1,float16,fp8,0,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,64,128,1,fp8,fp8,0,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,64,0,1,float16,fp8,0,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,64,0,1,float16,fp8,0,0.0124746672809124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,64,0,1,fp8,fp8,0,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,64,128,1,float16,float16,0,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,64,0,1,float16,float16,0,0.011946666985750198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,64,128,1,float16,fp8,0,0.012400000045696894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,64,128,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,64,0,1,fp8,fp8,0,0.022117334107557934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,64,128,1,float16,float16,0,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,64,0,1,float16,float16,0,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,64,128,1,float16,fp8,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,64,128,1,fp8,fp8,0,0.018645333747069042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,64,0,1,float16,fp8,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,64,0,1,fp8,fp8,0,0.01781333362062772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,64,128,1,float16,float16,0,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,64,128,1,float16,fp8,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,64,128,1,fp8,fp8,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,64,0,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,64,0,1,fp8,fp8,0,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,64,128,1,float16,float16,0,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,64,0,1,float16,float16,0,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,64,128,1,float16,fp8,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,64,128,1,fp8,fp8,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,64,0,1,float16,fp8,0,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,64,0,1,fp8,fp8,0,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,64,128,1,float16,float16,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,64,0,1,float16,float16,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,64,128,1,float16,fp8,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,64,128,1,fp8,fp8,0,0.018016000588734944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,64,0,1,float16,fp8,0,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,64,0,1,fp8,fp8,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,64,128,1,float16,float16,0,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,64,0,1,float16,float16,0,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,64,128,1,float16,fp8,0,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,64,128,1,fp8,fp8,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,64,0,1,float16,fp8,0,0.011920000116030375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,64,0,1,fp8,fp8,0,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,64,128,1,float16,float16,0,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,64,0,1,float16,float16,0,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,64,128,1,float16,fp8,0,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,64,128,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,64,0,1,float16,fp8,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,64,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,64,128,1,float16,float16,0,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,64,0,1,float16,float16,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,64,128,1,float16,fp8,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,64,128,1,fp8,fp8,0,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,64,0,1,float16,fp8,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,64,0,1,fp8,fp8,0,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,64,128,1,float16,float16,0,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,64,0,1,float16,float16,0,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,64,128,1,float16,fp8,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,64,128,1,fp8,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,64,0,1,float16,fp8,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,64,0,1,fp8,fp8,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,64,128,1,float16,float16,0,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,64,0,1,float16,float16,0,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,64,128,1,float16,fp8,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,64,128,1,fp8,fp8,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,64,0,1,float16,fp8,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,64,0,1,fp8,fp8,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,64,128,1,float16,float16,0,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,64,0,1,float16,float16,0,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,64,128,1,float16,fp8,0,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,64,128,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,64,0,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,64,0,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,64,128,1,float16,float16,0,4.386677424112956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,64,128,1,float16,fp8,0,4.375989278157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,64,128,1,fp8,fp8,0,5.543621063232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,64,128,1,float16,float16,0,4.564714749654134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,64,128,1,float16,fp8,0,4.532586733500163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,64,128,1,fp8,fp8,0,5.699743906656901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,64,128,1,float16,float16,0,4.575471878051758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,64,128,1,float16,fp8,0,4.525285402933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,64,0,1,fp8,fp8,0,25.591941833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,64,128,1,fp8,fp8,0,5.695685068766276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,64,0,1,float16,float16,0,28.46301778157552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,64,0,1,float16,fp8,0,28.721145629882812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,64,0,1,fp8,fp8,0,25.770469665527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,64,0,1,float16,float16,0,28.983060201009113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,64,128,1,float16,float16,0,4.900149345397949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,64,0,1,float16,fp8,0,31.20551045735677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,64,128,1,float16,fp8,0,4.568629264831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,64,128,1,fp8,fp8,0,5.765680313110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,64,0,1,float16,float16,0,29.99475606282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,64,128,1,float16,float16,0,2.6775999069213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,64,128,1,float16,fp8,0,2.554170608520508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,64,128,1,fp8,fp8,0,3.224837303161621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,64,0,1,fp8,fp8,0,26.139083862304688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,64,128,1,float16,float16,0,2.1826133728027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,64,0,1,float16,float16,0,14.661088307698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,64,0,1,float16,fp8,0,28.43413798014323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,64,128,1,float16,fp8,0,2.1245813369750977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,64,0,1,fp8,fp8,0,13.331050872802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,64,128,1,fp8,fp8,0,2.798309326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,64,0,1,float16,float16,0,29.258880615234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,64,0,1,float16,fp8,0,14.822090148925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,64,0,1,fp8,fp8,0,25.865147908528645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,64,128,1,float16,float16,0,2.228559970855713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,64,128,1,float16,fp8,0,2.2291359901428223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,64,0,1,float16,fp8,0,29.201433817545574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,64,128,1,fp8,fp8,0,2.8561652501424155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,64,128,1,float16,float16,0,2.276165326436361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,64,128,1,float16,fp8,0,2.2417333920796714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,64,0,1,float16,float16,0,14.38421376546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,64,0,1,fp8,fp8,0,12.985771179199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,64,128,1,fp8,fp8,0,2.8994932174682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,64,0,1,float16,fp8,0,15.47598393758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,64,128,1,float16,float16,0,2.2810932795206704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,64,0,1,float16,float16,0,16.109695434570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,64,0,1,fp8,fp8,0,12.957664489746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,64,0,1,float16,fp8,0,14.586831410725912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,64,128,1,float16,fp8,0,2.260986646016439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,64,128,1,fp8,fp8,0,2.9353440602620444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,64,128,1,float16,float16,0,1.2700533072153728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,64,0,1,float16,float16,0,14.706778208414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,64,128,1,float16,fp8,0,1.257034699122111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,64,128,1,fp8,fp8,0,1.6410667101542156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,64,0,1,float16,fp8,0,15.367552439371744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,64,0,1,fp8,fp8,0,13.30844243367513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,64,0,1,float16,float16,0,7.476565043131511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,64,128,1,float16,float16,0,1.1754559675852458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,64,0,1,float16,float16,0,14.555680592854818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,64,128,1,float16,fp8,0,1.1417333285013835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,64,0,1,float16,fp8,0,7.697706858317058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,64,0,1,fp8,fp8,0,6.715232213338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,64,128,1,fp8,fp8,0,1.4793492952982585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,64,0,1,fp8,fp8,0,13.342795054117838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,64,128,1,float16,float16,0,1.1665813128153484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,64,128,1,float16,fp8,0,1.149242639541626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,64,0,1,float16,fp8,0,14.558016459147135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,64,128,1,fp8,fp8,0,1.4680693944295247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,64,128,1,float16,float16,0,1.1618666648864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,64,0,1,float16,float16,0,7.254255930582683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,64,128,1,float16,fp8,0,1.1491680145263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,64,0,1,fp8,fp8,0,6.537946701049805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,64,0,1,float16,fp8,0,7.386501312255859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,64,128,1,fp8,fp8,0,1.4809759457906086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,64,0,1,float16,float16,0,7.829466501871745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,64,128,1,float16,float16,0,1.1645600001017253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,64,0,1,fp8,fp8,0,6.535706837972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,64,0,1,float16,fp8,0,7.212490717569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,64,128,1,float16,fp8,0,1.1965173085530598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,64,128,1,fp8,fp8,0,1.4841012954711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,64,128,1,float16,float16,0,0.8174293041229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,64,0,1,float16,float16,0,7.846362431844075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,64,128,1,float16,fp8,0,0.8180853525797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,64,128,1,fp8,fp8,0,1.0038080215454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,64,0,1,float16,fp8,0,7.219877243041992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,64,0,1,fp8,fp8,0,6.554943720499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,64,128,1,float16,float16,0,0.8172799746195475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,64,0,1,float16,float16,0,3.9644800821940103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,64,0,1,float16,float16,0,7.26529057820638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,64,128,1,float16,fp8,0,0.8171947002410889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,64,0,1,fp8,fp8,0,6.568090438842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,64,0,1,float16,fp8,0,7.1975250244140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,64,0,1,float16,fp8,0,3.9277865091959634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,64,128,1,fp8,fp8,0,0.991546630859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,64,0,1,fp8,fp8,0,3.5499467849731445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,64,128,1,float16,float16,0,0.8178133169809977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,64,0,1,float16,float16,0,3.875706672668457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,64,128,1,float16,fp8,0,0.817903995513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,64,128,1,fp8,fp8,0,0.9750080108642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,64,128,1,float16,float16,0,0.8177226384480795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,64,0,1,float16,fp8,0,3.8831892013549805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,64,0,1,fp8,fp8,0,3.5378452936808267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,64,0,1,float16,float16,0,4.014330546061198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,64,128,1,float16,fp8,0,0.8180267016092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,64,128,1,fp8,fp8,0,0.9774773120880127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,64,0,1,fp8,fp8,0,3.5386133193969727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,64,0,1,float16,fp8,0,3.984309196472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,64,128,1,float16,float16,0,0.8177973429361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,64,0,1,float16,float16,0,3.995567957560221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,64,128,1,float16,fp8,0,0.8176000118255615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,64,128,1,fp8,fp8,0,0.9796106815338135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,64,0,1,float16,fp8,0,3.8579893112182617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,64,0,1,fp8,fp8,0,3.5398292541503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,64,0,1,float16,float16,0,4.010304133097331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,64,0,1,float16,fp8,0,3.870453198750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,64,0,1,fp8,fp8,0,3.5416959126790366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,64,128,1,float16,float16,0,3.2636054356892905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,64,128,1,float16,fp8,0,3.217071851094564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,64,128,1,fp8,fp8,0,4.147695859273274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,64,128,1,float16,fp8,0,3.448810577392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,64,128,1,float16,float16,0,3.421253204345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,64,128,1,fp8,fp8,0,4.363114674886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,64,128,1,float16,float16,0,3.4345439275105796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,64,128,1,float16,fp8,0,3.380714734395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,64,0,1,float16,float16,0,16.97753651936849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,64,0,1,fp8,fp8,0,14.945215861002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,64,0,1,float16,fp8,0,17.173024495442707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,64,0,1,float16,float16,0,16.59162139892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,64,128,1,fp8,fp8,0,4.2511412302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,64,0,1,fp8,fp8,0,15.085968017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,64,0,1,float16,fp8,0,16.604042053222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,64,128,1,float16,float16,0,3.4692373275756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,64,128,1,float16,fp8,0,3.408522605895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,64,128,1,float16,float16,0,1.9080212910970051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,64,0,1,float16,float16,0,16.657354990641277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,64,128,1,fp8,fp8,0,4.336448033650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,64,128,1,float16,fp8,0,1.8677387237548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,64,128,1,fp8,fp8,0,2.4178454081217446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,64,0,1,fp8,fp8,0,15.099482218424479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,64,128,1,float16,float16,0,1.5916320482889812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,64,0,1,float16,fp8,0,18.27998987833659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,64,0,1,float16,float16,0,8.601055781046549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,64,128,1,float16,fp8,0,1.7093812624613445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,64,0,1,fp8,fp8,0,7.856378555297852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,64,128,1,fp8,fp8,0,2.0626559257507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,64,0,1,float16,fp8,0,8.922405242919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,64,0,1,float16,float16,0,17.88982899983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,64,0,1,fp8,fp8,0,15.746682484944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,64,128,1,float16,float16,0,1.6302186648050945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,64,128,1,float16,fp8,0,1.6168373425801594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,64,0,1,float16,fp8,0,18.80348841349284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,64,128,1,fp8,fp8,0,2.1076480547587075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,64,128,1,float16,float16,0,1.6549332936604817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,64,0,1,float16,float16,0,8.935493469238281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,64,128,1,float16,fp8,0,1.6129013697306316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,64,0,1,fp8,fp8,0,7.70033073425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,64,0,1,float16,fp8,0,8.307472229003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,64,128,1,fp8,fp8,0,2.135866641998291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,64,0,1,float16,float16,0,8.510357538859049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,64,128,1,float16,float16,0,1.673232078552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,64,0,1,fp8,fp8,0,7.5226186116536455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,64,0,1,float16,fp8,0,8.272768020629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,64,128,1,float16,fp8,0,1.7003413836161296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,64,0,1,float16,float16,0,8.356976191202799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,64,128,1,fp8,fp8,0,2.1657546361287436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,64,128,1,float16,float16,0,0.9535946846008301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,64,128,1,float16,fp8,0,0.9585546652475992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,64,128,1,fp8,fp8,0,1.2182880242665608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,64,0,1,fp8,fp8,0,7.565615971883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,64,0,1,float16,fp8,0,8.349050521850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,64,128,1,float16,float16,0,0.8725173473358154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,64,0,1,float16,float16,0,4.631690661112468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,64,0,1,float16,float16,0,8.406170527140299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,64,128,1,float16,fp8,0,0.8623147010803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,64,0,1,fp8,fp8,0,7.808586756388347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,64,0,1,fp8,fp8,0,3.9582506815592446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,64,128,1,fp8,fp8,0,1.1406933466593425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,64,0,1,float16,fp8,0,4.739514668782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,64,0,1,float16,fp8,0,8.329322814941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,64,128,1,float16,float16,0,0.8776053587595621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,64,128,1,float16,fp8,0,0.8837493260701498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,64,128,1,fp8,fp8,0,1.1709493001302083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,64,0,1,float16,float16,0,4.307450612386067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,64,128,1,float16,float16,0,0.8817493120829264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,64,128,1,float16,fp8,0,0.8754506905873617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,64,0,1,fp8,fp8,0,3.850384076436361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,64,0,1,float16,fp8,0,4.582810719807942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,64,128,1,fp8,fp8,0,1.1226933002471924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,64,0,1,float16,float16,0,4.475557327270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,64,128,1,float16,float16,0,0.9195413589477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,64,0,1,float16,fp8,0,4.31876786549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,64,0,1,fp8,fp8,0,3.847615877787272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,64,128,1,float16,fp8,0,0.8969386418660482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,64,0,1,float16,float16,0,4.363813400268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,64,128,1,fp8,fp8,0,1.1312639713287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,64,128,1,float16,float16,0,0.6228373448053995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,64,0,1,fp8,fp8,0,4.029786745707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,64,0,1,float16,fp8,0,4.558037439982097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,64,128,1,float16,fp8,0,0.6201599836349487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,64,128,1,fp8,fp8,0,0.7444480260213217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,64,0,1,float16,float16,0,4.474362691243489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,64,0,1,float16,float16,0,2.311290740966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,64,0,1,fp8,fp8,0,3.8613014221191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,64,0,1,float16,fp8,0,4.451098759969075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,64,128,1,float16,float16,0,0.618773341178894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,64,128,1,float16,fp8,0,0.619973341623942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,64,128,1,fp8,fp8,0,0.7420053482055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,64,0,1,fp8,fp8,0,2.136949380238851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,64,0,1,float16,fp8,0,2.3284533818562827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,64,128,1,float16,float16,0,0.6207413276036581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,64,0,1,float16,float16,0,2.3436479568481445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,64,128,1,float16,fp8,0,0.6193333466847738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,64,0,1,fp8,fp8,0,2.130224068959554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,64,0,1,float16,fp8,0,2.331472078959147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,64,128,1,fp8,fp8,0,0.7455413341522217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,64,128,1,float16,float16,0,0.6198773384094238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,64,128,1,float16,fp8,0,0.6207040150960287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,64,0,1,float16,float16,0,2.3053919474283853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,64,128,1,fp8,fp8,0,0.7461866537729899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,64,0,1,fp8,fp8,0,2.131594657897949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,64,0,1,float16,fp8,0,2.343839963277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,64,128,1,float16,float16,0,0.6204853455225626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,64,0,1,float16,float16,0,2.315290609995524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,64,128,1,float16,fp8,0,0.6196586688359579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,64,128,1,fp8,fp8,0,0.749018669128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,64,0,1,float16,fp8,0,2.31222931543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,64,0,1,fp8,fp8,0,2.1194559733072915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,64,0,1,float16,float16,0,2.315039952596029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,64,0,1,float16,fp8,0,2.312847932179769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,64,0,1,fp8,fp8,0,2.1478986740112305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,64,128,1,float16,float16,0,2.67906125386556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,64,128,1,float16,fp8,0,2.702218691507975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,64,128,1,fp8,fp8,0,3.504261334737142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,64,128,1,float16,float16,0,2.8201545079549155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,64,128,1,float16,fp8,0,2.8127946853637695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,64,128,1,fp8,fp8,0,3.579061190287272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,64,128,1,float16,float16,0,2.8422133127848306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,64,0,1,fp8,fp8,0,10.672618865966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,64,128,1,float16,fp8,0,2.824730555216471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,64,0,1,float16,float16,0,11.810719807942709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,64,0,1,float16,fp8,0,11.715141296386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,64,0,1,float16,float16,0,12.141989390055338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,64,128,1,fp8,fp8,0,3.565178553263346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,64,0,1,fp8,fp8,0,10.794661204020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,64,128,1,float16,float16,0,2.8636906941731772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,64,0,1,float16,fp8,0,11.89239501953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,64,128,1,float16,fp8,0,2.8494186401367188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,64,0,1,float16,float16,0,11.885786692301432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,64,128,1,fp8,fp8,0,3.591477394104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,64,128,1,float16,float16,0,1.5795307159423828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,64,128,1,float16,fp8,0,1.5286186536153157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,64,128,1,fp8,fp8,0,1.990517298380534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,64,0,1,fp8,fp8,0,10.807173411051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,64,0,1,float16,fp8,0,11.981664021809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,64,128,1,float16,float16,0,1.327290693918864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,64,0,1,float16,float16,0,6.125765482584636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,64,128,1,float16,fp8,0,1.3060213724772136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,64,0,1,float16,float16,0,12.771536509195963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,64,0,1,fp8,fp8,0,10.84930674235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,64,128,1,fp8,fp8,0,1.7184054056803386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,64,0,1,float16,fp8,0,6.264581044514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,64,0,1,fp8,fp8,0,5.614735921223958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,64,0,1,float16,fp8,0,11.987434387207031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,64,128,1,float16,float16,0,1.3400319417317708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,64,128,1,float16,fp8,0,1.3377973238627117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,64,0,1,float16,float16,0,6.24397341410319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,64,128,1,fp8,fp8,0,1.7459306716918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,64,128,1,float16,float16,0,1.3598186175028484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,64,128,1,float16,fp8,0,1.333194653193156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,64,0,1,fp8,fp8,0,5.69428825378418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,64,0,1,float16,fp8,0,6.488746643066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,64,128,1,fp8,fp8,0,1.7564959526062012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,64,0,1,float16,float16,0,6.073818842569987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,64,0,1,fp8,fp8,0,5.52183469136556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,64,128,1,float16,float16,0,1.3793813387552898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,64,0,1,float16,fp8,0,6.2199147542317705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,64,128,1,float16,fp8,0,1.3579200108846028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,64,128,1,float16,float16,0,0.801584005355835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,64,0,1,float16,float16,0,6.247664133707683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,64,128,1,fp8,fp8,0,1.837205410003662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,64,128,1,float16,fp8,0,0.8177013397216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,64,0,1,fp8,fp8,0,5.462725321451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,64,128,1,fp8,fp8,0,1.0216000080108643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,64,0,1,float16,fp8,0,5.916271845499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,64,128,1,float16,float16,0,0.7271626790364584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,64,0,1,float16,float16,0,5.898170471191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,64,0,1,float16,float16,0,3.3724212646484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,64,128,1,float16,fp8,0,0.7228693167368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,64,0,1,fp8,fp8,0,5.42578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,64,128,1,fp8,fp8,0,0.9414880275726318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,64,0,1,float16,fp8,0,5.906911849975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,64,0,1,fp8,fp8,0,2.8613173166910806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,64,0,1,float16,fp8,0,3.3733011881510415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,64,128,1,float16,float16,0,0.7334506511688232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,64,0,1,float16,float16,0,3.045210520426432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,64,128,1,float16,fp8,0,0.726032018661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,64,128,1,fp8,fp8,0,0.9374132951100668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,64,128,1,float16,float16,0,0.7363466421763102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,64,0,1,fp8,fp8,0,2.7626558939615884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,64,0,1,float16,fp8,0,3.259669303894043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,64,0,1,float16,float16,0,3.1456480026245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,64,128,1,float16,fp8,0,0.7268906434377035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,64,128,1,fp8,fp8,0,0.9301013151804606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,64,0,1,fp8,fp8,0,2.764575958251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,64,0,1,float16,fp8,0,3.0857439041137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,64,128,1,float16,float16,0,0.7403519948323568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,64,128,1,float16,fp8,0,0.7311360041300455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,64,0,1,float16,float16,0,3.342992146809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,64,128,1,fp8,fp8,0,0.9508372942606608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,64,0,1,float16,fp8,0,3.110015869140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,64,128,1,float16,float16,0,0.5369706551233927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,64,0,1,fp8,fp8,0,2.7724692026774087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,64,128,1,float16,fp8,0,0.5216426849365234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,64,128,1,fp8,fp8,0,0.6397706667582194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,64,0,1,float16,float16,0,3.3470398585001626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,64,0,1,float16,float16,0,1.6812106768290203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,64,128,1,float16,float16,0,0.5227733453114828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,64,0,1,float16,fp8,0,3.1960906982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,64,0,1,fp8,fp8,0,2.767242749532064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,64,128,1,float16,fp8,0,0.5206826527913412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,64,0,1,float16,fp8,0,1.71396271387736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,64,0,1,fp8,fp8,0,1.5599625905354817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,64,128,1,fp8,fp8,0,0.6291199922561646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,64,128,1,float16,float16,0,0.525050679842631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,64,0,1,float16,float16,0,1.7220160166422527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,64,128,1,float16,fp8,0,0.5205653508504232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,64,128,1,fp8,fp8,0,0.625327984491984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,64,0,1,float16,fp8,0,1.6875786781311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,64,0,1,fp8,fp8,0,1.5514987309773762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,64,128,1,float16,float16,0,0.5205066601435343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,64,0,1,float16,float16,0,1.7105813026428223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,64,128,1,float16,fp8,0,0.5201919873555502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,64,0,1,float16,fp8,0,1.7099092801411946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,64,128,1,fp8,fp8,0,0.6233866612116495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,64,0,1,fp8,fp8,0,1.5567359924316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,64,128,1,float16,float16,0,0.5203573306401571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,64,0,1,float16,float16,0,1.6915465990702312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,64,128,1,float16,fp8,0,0.5203306674957275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,64,128,1,fp8,fp8,0,0.6260799964269003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,64,0,1,float16,fp8,0,1.6848212877909343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,64,0,1,fp8,fp8,0,1.5573919614156086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,64,0,1,float16,float16,0,1.6837973594665527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,64,0,1,float16,fp8,0,1.6850454012552898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,64,0,1,fp8,fp8,0,1.557088057200114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,64,128,1,float16,float16,0,4.338234583536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,64,128,1,float16,fp8,0,4.302816073099772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,64,128,1,fp8,fp8,0,5.529359817504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,64,128,1,float16,float16,0,4.528031984965007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,64,128,1,float16,fp8,0,4.487119992574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,64,128,1,fp8,fp8,0,5.611258824666341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,64,128,1,float16,float16,0,4.532421429951985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,64,0,1,fp8,fp8,0,14.160357157389322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,64,0,1,float16,float16,0,15.586922963460287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,64,0,1,float16,fp8,0,16.021610260009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,64,0,1,float16,float16,0,15.901402791341146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,64,128,1,float16,fp8,0,4.495888074239095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,64,128,1,fp8,fp8,0,5.640410741170247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,64,0,1,fp8,fp8,0,14.343130747477213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,64,128,1,float16,float16,0,4.565605481465657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,64,0,1,float16,fp8,0,15.666272481282553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,64,128,1,float16,fp8,0,4.529386520385742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,64,0,1,float16,float16,0,15.786384582519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,64,128,1,fp8,fp8,0,5.675098419189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,64,128,1,float16,float16,0,2.5473333994547525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,64,128,1,float16,fp8,0,2.4788640340169272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,64,0,1,fp8,fp8,0,14.843711853027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,64,128,1,fp8,fp8,0,3.1611626942952475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,64,0,1,float16,fp8,0,17.52617009480794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,64,128,1,float16,float16,0,2.091397285461426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,64,0,1,float16,float16,0,8.570773442586264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,64,0,1,float16,float16,0,16.34652328491211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,64,0,1,fp8,fp8,0,14.884021759033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,64,128,1,float16,fp8,0,2.062122662862142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,64,128,1,fp8,fp8,0,2.7154293060302734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,64,0,1,fp8,fp8,0,7.616847991943359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,64,0,1,float16,fp8,0,8.50700823465983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,64,0,1,float16,fp8,0,16.342074076334637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,64,128,1,float16,float16,0,2.200965404510498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,64,0,1,float16,float16,0,8.016069412231445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,64,128,1,float16,fp8,0,2.1959253946940103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,64,128,1,fp8,fp8,0,2.8521226247151694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,64,128,1,float16,float16,0,2.227450688680013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,64,0,1,float16,fp8,0,8.463935852050781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,64,128,1,float16,fp8,0,2.2161812782287598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,64,0,1,fp8,fp8,0,7.282186508178711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,64,128,1,fp8,fp8,0,2.8579788208007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,64,0,1,float16,float16,0,8.00978151957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,64,0,1,float16,fp8,0,7.967530568440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,64,0,1,fp8,fp8,0,7.196661631266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,64,128,1,float16,float16,0,2.264992078145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,64,128,1,float16,fp8,0,2.241205374399821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,64,0,1,float16,float16,0,8.089616139729818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,64,128,1,fp8,fp8,0,2.8544638951619468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,64,128,1,float16,float16,0,1.2498239676157634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,64,0,1,fp8,fp8,0,7.220671971638997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,64,128,1,float16,fp8,0,1.2054346402486165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,64,0,1,float16,fp8,0,8.434730529785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,64,128,1,fp8,fp8,0,1.5644639333089192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,64,0,1,float16,float16,0,4.257733345031738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,64,0,1,fp8,fp8,0,7.244373321533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,64,0,1,float16,float16,0,8.030789057413736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,64,128,1,float16,float16,0,1.0663893222808838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,64,0,1,float16,fp8,0,8.315397262573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,64,128,1,float16,fp8,0,1.048634688059489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,64,0,1,float16,fp8,0,4.256346702575684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,64,128,1,fp8,fp8,0,1.3889066378275554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,64,0,1,fp8,fp8,0,3.7702560424804688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,64,128,1,float16,float16,0,1.0699466864267986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,64,128,1,float16,fp8,0,1.0624106725056965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,64,0,1,float16,float16,0,4.118560155232747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,64,128,1,fp8,fp8,0,1.3972800572713215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,64,0,1,float16,fp8,0,4.252458572387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,64,0,1,fp8,fp8,0,3.5779892603556314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,64,128,1,float16,float16,0,1.0939520200093586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,64,128,1,float16,fp8,0,1.119861364364624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,64,0,1,float16,float16,0,4.006677309672038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,64,128,1,fp8,fp8,0,1.401535987854004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,64,0,1,float16,fp8,0,3.944687843322754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,64,0,1,fp8,fp8,0,3.5652427673339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,64,128,1,float16,float16,0,1.116762638092041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,64,128,1,float16,fp8,0,1.0783519744873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,64,0,1,float16,float16,0,4.0801652272542315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,64,128,1,fp8,fp8,0,1.4180372556050618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,64,128,1,float16,float16,0,0.6438719828923544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,64,0,1,float16,fp8,0,4.367952028910319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,64,128,1,float16,fp8,0,0.6468319892883301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,64,0,1,fp8,fp8,0,3.5955947240193686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,64,128,1,fp8,fp8,0,0.8183093070983887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,64,0,1,float16,float16,0,4.109221458435059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,64,0,1,float16,float16,0,2.1571253140767417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,64,128,1,float16,float16,0,0.5897493362426758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,64,128,1,float16,fp8,0,0.6165973345438639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,64,0,1,float16,fp8,0,4.123066584269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,64,0,1,fp8,fp8,0,3.6084105173746743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,64,128,1,fp8,fp8,0,0.7740000089009603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,64,0,1,fp8,fp8,0,1.9126879374186199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,64,0,1,float16,fp8,0,2.1146933237711587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,64,0,1,float16,float16,0,2.023205280303955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,64,128,1,float16,float16,0,0.6236746708552042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,64,128,1,float16,fp8,0,0.5862400134404501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,64,0,1,float16,fp8,0,2.014421304066976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,64,0,1,fp8,fp8,0,1.8961706161499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,64,128,1,fp8,fp8,0,0.7546613216400146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,64,128,1,float16,float16,0,0.610373338063558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,64,0,1,float16,float16,0,2.0230560302734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,64,128,1,float16,fp8,0,0.614682674407959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,64,128,1,fp8,fp8,0,0.7603999773661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,64,0,1,fp8,fp8,0,1.8974080085754395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,64,0,1,float16,fp8,0,2.0182560284932456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,64,128,1,float16,float16,0,0.598309318224589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,64,128,1,float16,fp8,0,0.5927146673202515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,64,0,1,float16,float16,0,2.0552213986714682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,64,0,1,float16,fp8,0,2.095248063405355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,64,128,1,fp8,fp8,0,0.7800213495890299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,64,128,1,float16,float16,0,0.42214401563008624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,64,0,1,fp8,fp8,0,1.8727733294169109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,64,0,1,float16,float16,0,2.074848016103109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,64,128,1,float16,fp8,0,0.4275840123494466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,64,128,1,fp8,fp8,0,0.5080266793568929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,64,0,1,float16,float16,0,1.1802559693654378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,64,128,1,float16,float16,0,0.4235519965489705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,64,0,1,fp8,fp8,0,1.8452906608581543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,64,0,1,float16,fp8,0,2.0482773780822754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,64,128,1,float16,fp8,0,0.42236800988515216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,64,0,1,float16,fp8,0,1.1683680216471355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,64,0,1,fp8,fp8,0,1.0345706939697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,64,128,1,fp8,fp8,0,0.5096799929936727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,64,128,1,float16,float16,0,0.42418134212493896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,64,0,1,float16,float16,0,1.1818506717681885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,64,0,1,fp8,fp8,0,1.0234826405843098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,64,0,1,float16,fp8,0,1.1553280353546143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,64,128,1,float16,fp8,0,0.42525335152943927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,64,0,1,float16,float16,0,1.1532320181528728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,64,128,1,fp8,fp8,0,0.5066080093383789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,64,128,1,float16,float16,0,0.42324264844258624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,64,128,1,float16,fp8,0,0.42193599541982013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,64,128,1,fp8,fp8,0,0.5115093390146891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,64,0,1,fp8,fp8,0,1.0321333408355713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,64,0,1,float16,fp8,0,1.167423963546753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,64,128,1,float16,float16,0,0.42349334557851154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,64,0,1,float16,float16,0,1.1623093287150066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,64,0,1,float16,fp8,0,1.1724747021993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,64,0,1,fp8,fp8,0,1.0343519846598308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,64,128,1,float16,fp8,0,0.42424531777699787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,64,128,1,fp8,fp8,0,0.5079733530680338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,64,0,1,float16,float16,0,1.156874656677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,64,0,1,fp8,fp8,0,1.047594706217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,64,0,1,float16,fp8,0,1.170741319656372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,64,128,1,float16,float16,0,3.2576427459716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,64,128,1,float16,fp8,0,3.1989545822143555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,64,128,1,float16,float16,0,3.3982718785603843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,64,128,1,fp8,fp8,0,4.096266746520996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,64,128,1,float16,fp8,0,3.3637653986612954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,64,128,1,fp8,fp8,0,4.214042663574219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,64,0,1,fp8,fp8,0,8.462026596069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,64,0,1,float16,float16,0,9.401034673055014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,64,0,1,float16,fp8,0,9.331482569376627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,64,128,1,float16,float16,0,3.399423917134603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,64,0,1,float16,float16,0,9.560789108276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,64,128,1,float16,fp8,0,3.3581600189208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,64,0,1,fp8,fp8,0,8.627045313517252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,64,0,1,float16,fp8,0,9.466538747151693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,64,128,1,fp8,fp8,0,4.228517214457194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,64,128,1,float16,float16,0,3.4244426091512046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,64,128,1,float16,fp8,0,3.4027414321899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,64,0,1,float16,float16,0,9.516714731852213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,64,128,1,fp8,fp8,0,4.239157358805339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,64,0,1,fp8,fp8,0,8.648506800333658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,64,0,1,float16,fp8,0,9.448037465413412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,64,128,1,float16,float16,0,1.8994240760803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,64,128,1,float16,fp8,0,1.8308960596720378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,64,128,1,fp8,fp8,0,2.37825600306193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,64,128,1,float16,float16,0,1.5385440190633137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,64,0,1,float16,float16,0,9.510117212931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,64,0,1,float16,float16,0,5.152746518452962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,64,128,1,float16,fp8,0,1.5143413543701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,64,0,1,fp8,fp8,0,4.588608105977376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,64,128,1,fp8,fp8,0,2.0195892651875815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,64,0,1,float16,fp8,0,4.946554819742839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,64,0,1,fp8,fp8,0,8.682986577351889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,64,0,1,float16,fp8,0,9.570677439371744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,64,128,1,float16,float16,0,1.577013333638509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,64,0,1,float16,float16,0,4.8109385172526045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,64,128,1,float16,fp8,0,1.5658987363179524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,64,0,1,fp8,fp8,0,4.215029398600261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,64,128,1,fp8,fp8,0,2.0368000666300454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,64,0,1,float16,fp8,0,4.847434679667155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,64,128,1,float16,float16,0,1.6183946927388508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,64,0,1,float16,float16,0,4.873189290364583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,64,128,1,float16,fp8,0,1.5616745948791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,64,128,1,fp8,fp8,0,2.059919993082682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,64,0,1,fp8,fp8,0,4.244570732116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,64,0,1,float16,fp8,0,5.070853233337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,64,128,1,float16,float16,0,1.6229333877563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,64,0,1,float16,float16,0,4.912517229715983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,64,128,1,float16,fp8,0,1.5792320569356282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,64,0,1,fp8,fp8,0,4.321642557779948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,64,128,1,float16,float16,0,0.9481013615926107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,64,128,1,fp8,fp8,0,2.093834718068441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,64,0,1,float16,fp8,0,5.165301322937012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,64,128,1,float16,fp8,0,0.9156266848246256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,64,128,1,fp8,fp8,0,1.1660959720611572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,64,0,1,float16,float16,0,4.952165285746257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,64,0,1,float16,float16,0,2.4666773478190103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,64,128,1,float16,float16,0,0.8132266998291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,64,0,1,float16,fp8,0,2.5101067225138345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,64,0,1,fp8,fp8,0,4.320528030395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,64,0,1,float16,fp8,0,4.7784372965494795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,64,0,1,fp8,fp8,0,2.3156587282816568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,64,128,1,float16,fp8,0,0.8237973054250082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,64,128,1,fp8,fp8,0,1.0813759962717693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,64,0,1,float16,float16,0,2.3478612899780273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,64,128,1,float16,float16,0,0.8347360293070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,64,128,1,float16,fp8,0,0.8261280059814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,64,128,1,fp8,fp8,0,1.0535519917805989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,64,0,1,float16,fp8,0,2.4132320086161294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,64,128,1,float16,float16,0,0.8205280303955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,64,0,1,fp8,fp8,0,2.202064037322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,64,128,1,float16,fp8,0,0.8108639717102051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,64,0,1,float16,float16,0,2.3738560676574707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,64,0,1,float16,fp8,0,2.3653386433919272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,64,128,1,fp8,fp8,0,1.06222931543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,64,0,1,fp8,fp8,0,2.1572799682617188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,64,128,1,float16,float16,0,0.8342453638712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,64,0,1,float16,float16,0,2.4144479433695474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,64,128,1,float16,fp8,0,0.8225173155466715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,64,128,1,fp8,fp8,0,1.0871893564860027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,64,128,1,float16,float16,0,0.498143990834554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,64,0,1,fp8,fp8,0,2.1641012827555337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,64,0,1,float16,fp8,0,2.3497653007507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,64,128,1,float16,fp8,0,0.5013440052668253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,64,0,1,float16,float16,0,2.399727980295817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,64,0,1,float16,float16,0,1.2670613129933674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,64,128,1,fp8,fp8,0,0.6269919872283936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,64,128,1,float16,float16,0,0.4589333136876424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,64,0,1,fp8,fp8,0,2.1725974082946777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,64,0,1,float16,fp8,0,2.4580532709757485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,64,128,1,float16,fp8,0,0.4447520176569621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,64,0,1,float16,fp8,0,1.2816533247629802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,64,0,1,fp8,fp8,0,1.2242826620737712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,64,128,1,fp8,fp8,0,0.5749546686808268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,64,128,1,float16,float16,0,0.4693439801534017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,64,0,1,float16,float16,0,1.2414933045705159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,64,128,1,float16,fp8,0,0.45480533440907794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,64,128,1,fp8,fp8,0,0.5885706742604574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,64,0,1,float16,fp8,0,1.2401386896769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,64,0,1,fp8,fp8,0,1.1338880062103271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,64,128,1,float16,float16,0,0.45446932315826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,64,128,1,float16,fp8,0,0.45679465929667157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,64,0,1,float16,float16,0,1.2439040342966716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,64,0,1,float16,fp8,0,1.22325865427653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,64,128,1,fp8,fp8,0,0.5876533190409342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,64,0,1,fp8,fp8,0,1.1585813363393147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,64,0,1,float16,float16,0,1.2553973197937012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,64,128,1,float16,float16,0,0.46530667940775555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,64,128,1,float16,fp8,0,0.4520266850789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,64,0,1,float16,fp8,0,1.2123946348826091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,64,0,1,fp8,fp8,0,1.135973294576009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,64,128,1,fp8,fp8,0,0.5923200050989786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,64,0,1,float16,float16,0,1.2483839988708496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,64,128,1,float16,float16,0,0.32418133815129596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,64,128,1,float16,fp8,0,0.32316267490386963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,64,128,1,fp8,fp8,0,0.3941546678543091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,64,0,1,float16,fp8,0,1.2539199988047283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,64,0,1,float16,float16,0,0.718069314956665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,64,0,1,fp8,fp8,0,1.1441226800282795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,64,128,1,float16,float16,0,0.3213333288828532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,64,0,1,float16,fp8,0,0.723861296971639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,64,0,1,fp8,fp8,0,0.6461653312047323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,64,128,1,float16,fp8,0,0.32529600461324054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,64,128,1,fp8,fp8,0,0.3858666817347209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,64,0,1,float16,float16,0,0.7398506800333658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,64,128,1,float16,float16,0,0.3222986658414205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,64,0,1,float16,fp8,0,0.7255733013153076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,64,0,1,fp8,fp8,0,0.6444693406422933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,64,128,1,float16,fp8,0,0.32531734307607013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,64,128,1,fp8,fp8,0,0.38706668217976886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,64,0,1,float16,float16,0,0.7243680159250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,64,128,1,float16,float16,0,0.3228800098101298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,64,0,1,fp8,fp8,0,0.6464960177739462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,64,128,1,float16,fp8,0,0.3233333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,64,0,1,float16,fp8,0,0.7386720180511475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,64,128,1,fp8,fp8,0,0.38702932993570965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,64,0,1,float16,float16,0,0.7244426409403483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,64,128,1,float16,float16,0,0.3226880033810933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,64,0,1,float16,fp8,0,0.7247040271759033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,64,0,1,fp8,fp8,0,0.6537280082702637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,64,128,1,float16,fp8,0,0.3261920015017192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,64,0,1,float16,float16,0,0.7264800071716309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,64,128,1,fp8,fp8,0,0.3868746757507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,64,0,1,float16,fp8,0,0.725488026936849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,64,0,1,fp8,fp8,0,0.6472693284352621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,64,128,1,float16,float16,0,4.3898773193359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,64,128,1,float16,fp8,0,4.283354759216309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,64,128,1,float16,float16,0,4.48416010538737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,64,128,1,fp8,fp8,0,5.420981089274089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,64,0,1,fp8,fp8,0,8.532400131225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,64,0,1,float16,float16,0,9.218714396158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,64,128,1,float16,fp8,0,4.468725204467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,64,0,1,float16,fp8,0,9.204645156860352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,64,128,1,fp8,fp8,0,5.595050811767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,64,128,1,float16,float16,0,4.497237205505371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,64,0,1,float16,float16,0,9.4693972269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,64,128,1,float16,fp8,0,4.469663937886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,64,0,1,fp8,fp8,0,8.561002731323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,64,0,1,float16,fp8,0,9.404607772827148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,64,128,1,fp8,fp8,0,5.586032231648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,64,128,1,float16,float16,0,4.530106544494629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,64,0,1,float16,float16,0,9.396533330281576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,64,128,1,float16,fp8,0,4.481578509012858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,64,0,1,float16,fp8,0,9.503184000651041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,64,128,1,fp8,fp8,0,5.640719731648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,64,128,1,float16,float16,0,2.5415627161661782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,64,0,1,fp8,fp8,0,8.629621505737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,64,128,1,float16,fp8,0,2.477952003479004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,64,0,1,float16,float16,0,9.534063975016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,64,0,1,float16,float16,0,4.995573361714681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,64,128,1,fp8,fp8,0,3.1216745376586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,64,0,1,float16,fp8,0,9.42466672261556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,64,128,1,float16,float16,0,2.064469337463379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,64,128,1,float16,fp8,0,2.0796106656392417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,64,0,1,float16,fp8,0,5.08460807800293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,64,0,1,fp8,fp8,0,8.638170878092447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,64,0,1,fp8,fp8,0,4.6484800974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,64,128,1,fp8,fp8,0,2.6495200792948403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,64,0,1,float16,float16,0,4.8253278732299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,64,128,1,float16,float16,0,2.2165493965148926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,64,0,1,fp8,fp8,0,4.184336026509603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,64,128,1,float16,fp8,0,2.162816047668457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,64,0,1,float16,fp8,0,4.72161070505778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,64,128,1,fp8,fp8,0,2.7860212326049805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,64,128,1,float16,float16,0,2.20907195409139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,64,0,1,float16,float16,0,4.8893280029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,64,128,1,float16,fp8,0,2.181114673614502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,64,0,1,float16,fp8,0,4.9183305104573565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,64,128,1,fp8,fp8,0,2.8489065170288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,64,0,1,fp8,fp8,0,4.313765207926433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,64,128,1,float16,float16,0,2.242981274922689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,64,0,1,float16,float16,0,4.904293378194173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,64,128,1,float16,fp8,0,2.2377707163492837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,64,0,1,float16,fp8,0,4.969248135884603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,64,0,1,fp8,fp8,0,4.329466819763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,64,128,1,fp8,fp8,0,2.854757308959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,64,128,1,float16,float16,0,1.2391200065612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,64,128,1,float16,fp8,0,1.2038133144378662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,64,0,1,float16,float16,0,4.920495986938477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,64,0,1,float16,float16,0,2.5032426516215005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,64,128,1,fp8,fp8,0,1.5408639907836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,64,0,1,float16,fp8,0,4.793456077575684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,64,128,1,float16,float16,0,1.0412906805674236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,64,0,1,float16,fp8,0,2.4353386561075845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,64,128,1,float16,fp8,0,1.024778683980306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,64,0,1,fp8,fp8,0,2.2975680033365884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,64,0,1,fp8,fp8,0,4.367130597432454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,64,128,1,fp8,fp8,0,1.3403679529825847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,64,0,1,float16,float16,0,2.2877066930135093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,64,0,1,fp8,fp8,0,2.0927200317382812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,64,128,1,float16,float16,0,1.0489760239919026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,64,0,1,float16,fp8,0,2.41813325881958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,64,128,1,float16,fp8,0,1.0357813040415447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,64,128,1,fp8,fp8,0,1.3312746683756511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,64,0,1,float16,float16,0,2.3211572964986167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,64,128,1,float16,float16,0,1.053434689839681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,64,128,1,float16,fp8,0,1.0417760213216145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,64,0,1,float16,fp8,0,2.2718613942464194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,64,0,1,fp8,fp8,0,2.1018773714701333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,64,128,1,fp8,fp8,0,1.336458683013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,64,0,1,float16,float16,0,2.466298739115397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,64,128,1,float16,float16,0,1.066373348236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,64,128,1,float16,fp8,0,1.0447626908620198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,64,0,1,fp8,fp8,0,2.1549545923868814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,64,0,1,float16,fp8,0,2.344693342844645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,64,128,1,fp8,fp8,0,1.3897066116333008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,64,0,1,float16,float16,0,2.3101332982381186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,64,128,1,float16,float16,0,0.6240959962209066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,64,128,1,float16,fp8,0,0.6189119815826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,64,128,1,fp8,fp8,0,0.7938613096872965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,64,0,1,fp8,fp8,0,2.1250720024108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,64,0,1,float16,float16,0,1.2711093425750732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,64,0,1,float16,fp8,0,2.41756804784139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,64,128,1,float16,float16,0,0.5508000055948893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,64,0,1,float16,fp8,0,1.2364532947540283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,64,128,1,float16,fp8,0,0.5412266651789347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,64,0,1,fp8,fp8,0,1.1519253253936768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,64,128,1,fp8,fp8,0,0.7111519972483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,64,0,1,float16,float16,0,1.193493366241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,64,0,1,float16,fp8,0,1.171072006225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,64,128,1,float16,float16,0,0.5550613403320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,64,128,1,float16,fp8,0,0.5405066808064779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,64,0,1,fp8,fp8,0,1.0872639815012615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,64,128,1,fp8,fp8,0,0.7149706681569418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,64,0,1,float16,float16,0,1.181536038716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,64,128,1,float16,float16,0,0.5554826656977335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,64,0,1,float16,fp8,0,1.186037302017212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,64,128,1,float16,fp8,0,0.5490719874699911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,64,0,1,fp8,fp8,0,1.090351978937785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,64,128,1,fp8,fp8,0,0.717408021291097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,64,0,1,float16,float16,0,1.1760533650716145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,64,128,1,float16,float16,0,0.5701013406117758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,64,0,1,fp8,fp8,0,1.0931306680043538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,64,128,1,float16,fp8,0,0.5532693465550741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,64,0,1,float16,fp8,0,1.1876320044199626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,64,128,1,fp8,fp8,0,0.721343994140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,64,128,1,float16,float16,0,0.34016533692677814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,64,0,1,float16,float16,0,1.2011946837107341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,64,128,1,float16,fp8,0,0.3376426696777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,64,128,1,fp8,fp8,0,0.43572266896565753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,64,0,1,float16,float16,0,0.6653386751810709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,64,0,1,float16,fp8,0,1.1831893126169841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,64,0,1,fp8,fp8,0,1.1097013155619304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,64,0,1,float16,fp8,0,0.6563040018081665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,64,128,1,float16,float16,0,0.30948267380396527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,64,0,1,fp8,fp8,0,0.6100426514943441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,64,128,1,float16,fp8,0,0.30853333075841266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,64,128,1,fp8,fp8,0,0.40115201473236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,64,0,1,float16,float16,0,0.6307413180669149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,64,128,1,float16,float16,0,0.3083466688791911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,64,0,1,float16,fp8,0,0.6323413451512655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,64,0,1,fp8,fp8,0,0.5706400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,64,128,1,float16,fp8,0,0.30913599332173664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,64,0,1,float16,float16,0,0.63045334815979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,64,128,1,fp8,fp8,0,0.40379734834035236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,64,128,1,float16,float16,0,0.3171413342158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,64,0,1,float16,fp8,0,0.635258674621582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,64,0,1,fp8,fp8,0,0.5654559930165609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,64,128,1,float16,fp8,0,0.310479998588562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,64,0,1,float16,float16,0,0.6363093455632528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,64,128,1,fp8,fp8,0,0.40040000279744464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,64,128,1,float16,float16,0,0.31890666484832764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,64,0,1,float16,fp8,0,0.6287360191345215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,64,0,1,fp8,fp8,0,0.5778079827626547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,64,128,1,float16,fp8,0,0.30948267380396527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,64,0,1,float16,float16,0,0.6448320150375366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,64,128,1,fp8,fp8,0,0.40196800231933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,64,128,1,float16,float16,0,0.22697067260742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,64,0,1,float16,fp8,0,0.6323306560516357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,64,0,1,fp8,fp8,0,0.5724053382873535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,64,128,1,float16,fp8,0,0.2253013253211975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,64,0,1,float16,float16,0,0.3752853473027547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,64,128,1,fp8,fp8,0,0.2741546630859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,64,0,1,float16,fp8,0,0.37907731533050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,64,0,1,fp8,fp8,0,0.3495519955952962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,64,128,1,float16,float16,0,0.22403200467427573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,64,128,1,float16,fp8,0,0.2227946718533834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,64,0,1,float16,float16,0,0.37041600545247394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,64,128,1,fp8,fp8,0,0.27146132787068683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,64,0,1,float16,fp8,0,0.3717919985453288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,64,0,1,fp8,fp8,0,0.34749333063761395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,64,128,1,float16,float16,0,0.22382400433222452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,64,128,1,float16,fp8,0,0.22402133544286093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,64,0,1,float16,float16,0,0.3707306782404582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,64,128,1,fp8,fp8,0,0.27187732855478924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,64,0,1,float16,fp8,0,0.3708213170369466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,64,0,1,fp8,fp8,0,0.34836800893147785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,64,128,1,float16,float16,0,0.22450133164723715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,64,0,1,float16,float16,0,0.3695840040842692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,64,128,1,float16,fp8,0,0.22435200214385986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,64,128,1,fp8,fp8,0,0.2725706696510315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,64,0,1,float16,fp8,0,0.37081066767374676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,64,128,1,float16,float16,0,0.2238346735636393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,64,0,1,fp8,fp8,0,0.34911465644836426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,64,128,1,float16,fp8,0,0.22457599639892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,64,0,1,float16,float16,0,0.37305064996083576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,64,128,1,fp8,fp8,0,0.27365867296854657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,64,0,1,float16,fp8,0,0.37213865915934247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,64,0,1,fp8,fp8,0,0.34860801696777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,64,128,1,float16,float16,0,3.130234718322754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,64,128,1,float16,fp8,0,3.103359858194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,64,128,1,float16,float16,0,3.2268638610839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,64,128,1,fp8,fp8,0,4.01474126180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,64,0,1,float16,float16,0,5.645301183064778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,64,0,1,fp8,fp8,0,5.182815869649251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,64,0,1,float16,fp8,0,5.701914469401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,64,128,1,float16,fp8,0,3.2013654708862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,64,128,1,fp8,fp8,0,4.032959938049316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,64,0,1,float16,float16,0,5.916389465332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,64,128,1,float16,float16,0,3.245354652404785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,64,128,1,float16,fp8,0,3.2246081034342446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,64,0,1,float16,fp8,0,5.715066909790039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,64,0,1,fp8,fp8,0,5.313125292460124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,64,128,1,fp8,fp8,0,4.0745439529418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,64,0,1,float16,float16,0,5.9059092203776045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,64,128,1,float16,float16,0,3.2799787521362305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,64,128,1,float16,fp8,0,3.261610666910807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,64,0,1,float16,fp8,0,5.863583882649739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,64,0,1,fp8,fp8,0,5.294021288553874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,64,128,1,float16,float16,0,1.8609493573506672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,64,128,1,fp8,fp8,0,4.148293177286784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,64,0,1,float16,float16,0,6.063695907592773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,64,128,1,float16,fp8,0,1.8259466489156086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,64,128,1,fp8,fp8,0,2.3281493186950684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,64,0,1,float16,float16,0,3.12989870707194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,64,0,1,float16,fp8,0,5.783871968587239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,64,0,1,fp8,fp8,0,5.41213862101237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,64,128,1,float16,float16,0,1.5324320793151855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,64,0,1,float16,fp8,0,3.1028213500976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,64,0,1,fp8,fp8,0,2.907594680786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,64,128,1,float16,fp8,0,1.4825119972229004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,64,128,1,float16,float16,0,1.5630559921264648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,64,128,1,fp8,fp8,0,2.0132853190104165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,64,0,1,float16,float16,0,2.8189919789632163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,64,0,1,fp8,fp8,0,2.5366293589274087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,64,0,1,float16,fp8,0,2.768821398417155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,64,128,1,float16,fp8,0,1.5152692794799805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,64,128,1,fp8,fp8,0,1.990890661875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,64,128,1,float16,float16,0,1.5629547437032063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,64,0,1,float16,float16,0,2.8284533818562827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,64,128,1,float16,fp8,0,1.5423520406087239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,64,0,1,fp8,fp8,0,2.6156105995178223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,64,0,1,float16,fp8,0,2.9032745361328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,64,128,1,fp8,fp8,0,1.9984745979309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,64,128,1,float16,float16,0,1.584661324818929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,64,0,1,float16,float16,0,3.061509450276693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,64,0,1,float16,fp8,0,2.8121652603149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,64,128,1,float16,fp8,0,1.571989377339681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,64,0,1,fp8,fp8,0,2.624330679575602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,64,128,1,float16,float16,0,0.9409600098927816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,64,128,1,fp8,fp8,0,2.0684213638305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,64,0,1,float16,float16,0,2.908458709716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,64,128,1,float16,fp8,0,0.8974239826202393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,64,128,1,fp8,fp8,0,1.1221919854482014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,64,0,1,float16,fp8,0,2.8335787455240884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,64,0,1,float16,float16,0,1.5851786931355794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,64,0,1,fp8,fp8,0,2.6449920336405435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,64,128,1,float16,float16,0,0.793013334274292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,64,0,1,float16,fp8,0,1.5369866689046223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,64,0,1,fp8,fp8,0,1.4304693539937336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,64,128,1,float16,fp8,0,0.7668106555938721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,64,128,1,fp8,fp8,0,1.0570186773935955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,64,0,1,float16,float16,0,1.4195626576741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,64,128,1,float16,float16,0,0.7908000151316324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,64,0,1,float16,fp8,0,1.4268746376037598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,64,0,1,fp8,fp8,0,1.3361066182454426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,64,128,1,float16,fp8,0,0.7780106862386068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,64,0,1,float16,float16,0,1.430250644683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,64,128,1,fp8,fp8,0,1.0058613618214924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,64,128,1,float16,float16,0,0.7937066555023193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,64,0,1,fp8,fp8,0,1.305898666381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,64,0,1,float16,fp8,0,1.4402186075846355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,64,128,1,float16,fp8,0,0.7975306510925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,64,128,1,fp8,fp8,0,1.0192320346832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,64,0,1,float16,float16,0,1.4313279787699382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,64,128,1,float16,float16,0,0.8014079729715983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,64,0,1,float16,fp8,0,1.4216480255126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,64,0,1,fp8,fp8,0,1.3071253299713135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,64,128,1,float16,fp8,0,0.7915679613749186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,64,128,1,fp8,fp8,0,1.0334346294403076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,64,128,1,float16,float16,0,0.4757066567738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,64,0,1,float16,float16,0,1.4291787147521973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,64,0,1,fp8,fp8,0,1.3277973333994548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,64,0,1,float16,fp8,0,1.4279200236002605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,64,128,1,float16,fp8,0,0.46590399742126465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,64,128,1,float16,float16,0,0.41786666711171466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,64,0,1,float16,float16,0,0.799232006072998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,64,128,1,fp8,fp8,0,0.593994657198588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,64,0,1,float16,fp8,0,0.793008009592692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,64,0,1,fp8,fp8,0,0.739904006322225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,64,128,1,float16,fp8,0,0.41283734639485675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,64,0,1,float16,float16,0,0.7431573073069254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,64,128,1,fp8,fp8,0,0.5336106618245443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,64,0,1,float16,fp8,0,0.7371093432108561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,64,128,1,float16,float16,0,0.4222240050633748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,64,0,1,fp8,fp8,0,0.6872853438059489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,64,128,1,float16,fp8,0,0.42445866266886395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,64,0,1,float16,float16,0,0.7442293167114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,64,128,1,fp8,fp8,0,0.5405813455581665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,64,128,1,float16,float16,0,0.4219199816385905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,64,0,1,float16,fp8,0,0.7340053717295328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,64,0,1,fp8,fp8,0,0.704266627629598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,64,128,1,float16,fp8,0,0.4259893496831258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,64,0,1,float16,float16,0,0.7483519713083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,64,128,1,fp8,fp8,0,0.544538656870524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,64,0,1,float16,fp8,0,0.7404212951660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,64,128,1,float16,float16,0,0.4215306838353475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,64,0,1,fp8,fp8,0,0.6917813618977865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,64,128,1,float16,fp8,0,0.4312586784362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,64,128,1,fp8,fp8,0,0.5482293367385864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,64,0,1,float16,float16,0,0.7663893699645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,64,128,1,float16,float16,0,0.25965332984924316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,64,0,1,float16,fp8,0,0.746346632639567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,64,128,1,float16,fp8,0,0.2616960008939107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,64,0,1,fp8,fp8,0,0.6956906318664551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,64,0,1,float16,float16,0,0.4250933329264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,64,128,1,fp8,fp8,0,0.3309599955876668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,64,0,1,float16,fp8,0,0.4268426497777303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,64,128,1,float16,float16,0,0.23694399992624918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,64,0,1,fp8,fp8,0,0.394159992535909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,64,128,1,float16,fp8,0,0.23470399777094522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,64,0,1,float16,float16,0,0.40907732645670575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,64,128,1,fp8,fp8,0,0.30542399485905963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,64,0,1,float16,fp8,0,0.4095199902852376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,64,0,1,fp8,fp8,0,0.36346666018168133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,64,128,1,float16,float16,0,0.23701866467793783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,64,128,1,float16,fp8,0,0.23452800512313843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,64,0,1,float16,float16,0,0.4099573294321696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,64,128,1,fp8,fp8,0,0.3086293339729309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,64,0,1,float16,fp8,0,0.4036373297373454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,64,0,1,fp8,fp8,0,0.37010133266448975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,64,128,1,float16,float16,0,0.2397759954134623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,64,0,1,float16,float16,0,0.4045279820760091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,64,128,1,float16,fp8,0,0.23431466023127237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,64,128,1,fp8,fp8,0,0.30952000617980957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,64,0,1,float16,fp8,0,0.40611732006073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,64,0,1,fp8,fp8,0,0.3633546829223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,64,128,1,float16,float16,0,0.24305067459742227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,64,128,1,float16,fp8,0,0.23795199394226074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,64,0,1,float16,float16,0,0.4085013469060262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,64,128,1,fp8,fp8,0,0.3126293420791626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,64,0,1,float16,fp8,0,0.4084320068359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,64,128,1,float16,float16,0,0.17547200123469034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,64,0,1,fp8,fp8,0,0.36721599102020264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,64,128,1,float16,fp8,0,0.17509865760803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,64,0,1,float16,float16,0,0.2519413431485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,64,128,1,fp8,fp8,0,0.2159093419710795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,64,0,1,float16,fp8,0,0.252890666325887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,64,0,1,fp8,fp8,0,0.23356266816457114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,64,128,1,float16,float16,0,0.17234132687250772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,64,128,1,float16,fp8,0,0.1739306648572286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,64,0,1,float16,float16,0,0.24685867627461752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,64,128,1,fp8,fp8,0,0.21237866083780924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,64,128,1,fp8,fp8,0,0.21271467208862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,64,0,1,float16,fp8,0,0.24764800071716309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,64,0,1,float16,fp8,0,0.24638400475184122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,64,0,1,fp8,fp8,0,0.23255467414855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,64,128,1,float16,float16,0,0.17387733856836954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,64,128,1,float16,fp8,0,0.17219199736913046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,64,0,1,float16,float16,0,0.24608532587687174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,64,0,1,fp8,fp8,0,0.23241599400838217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,64,128,1,float16,float16,0,0.17298134167989096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,64,0,1,float16,float16,0,0.246943990389506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,64,128,1,float16,float16,0,0.17413866519927979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,64,128,1,float16,fp8,0,0.1734453241030375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,64,128,1,fp8,fp8,0,0.2132426699002584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,64,128,1,float16,fp8,0,0.17358932892481485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,64,0,1,float16,fp8,0,0.24868800242741904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,64,0,1,fp8,fp8,0,0.23228265841801962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,64,0,1,float16,float16,0,0.2457653284072876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,64,128,1,fp8,fp8,0,0.21321600675582886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,64,0,1,float16,fp8,0,0.2485333283742269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,64,0,1,fp8,fp8,0,0.23292267322540283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,64,128,1,float16,float16,0,4.191935857137044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,64,128,1,float16,fp8,0,4.149269421895345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,64,128,1,float16,float16,0,4.207141240437825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,64,128,1,fp8,fp8,0,5.170682589213054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,64,0,1,float16,float16,0,6.063776016235352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,64,0,1,fp8,fp8,0,5.400282541910808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,64,0,1,float16,fp8,0,5.995136260986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,64,0,1,float16,float16,0,6.051392237345378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,64,128,1,float16,fp8,0,4.156240145365397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,64,128,1,fp8,fp8,0,5.3304853439331055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,64,128,1,float16,float16,0,4.238325436909993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,64,128,1,float16,fp8,0,4.153093338012695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,64,0,1,fp8,fp8,0,5.546511967976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,64,0,1,float16,fp8,0,6.0022398630778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,64,0,1,float16,float16,0,6.251263936360677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,64,128,1,fp8,fp8,0,5.318239847819011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,64,0,1,float16,fp8,0,6.036458969116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,64,128,1,float16,float16,0,4.3355147043863935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,64,0,1,fp8,fp8,0,5.53163210550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,64,128,1,float16,fp8,0,4.250629425048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,64,0,1,float16,float16,0,6.17037836710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,64,128,1,float16,float16,0,2.464303970336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,64,128,1,fp8,fp8,0,5.464480082194011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,64,128,1,float16,fp8,0,2.391786734263102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,64,0,1,float16,fp8,0,6.117317199707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,64,0,1,float16,float16,0,3.3908958435058594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,64,128,1,fp8,fp8,0,3.0285441080729165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,64,128,1,float16,float16,0,2.009082635243734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,64,0,1,fp8,fp8,0,5.6084855397542315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,64,0,1,fp8,fp8,0,3.1441866556803384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,64,0,1,float16,fp8,0,3.351349194844564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,64,128,1,float16,fp8,0,1.9852852821350098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,64,0,1,float16,float16,0,2.94977601369222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,64,128,1,fp8,fp8,0,2.561807950337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,64,0,1,fp8,fp8,0,2.6685492197672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,64,0,1,float16,fp8,0,2.924821217854818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,64,128,1,float16,float16,0,2.0524373054504395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,64,128,1,float16,fp8,0,2.0325172742207847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,64,128,1,fp8,fp8,0,2.6439946492513022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,64,0,1,float16,float16,0,3.0425281524658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,64,0,1,float16,fp8,0,2.978399912516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,64,128,1,float16,float16,0,2.07043727238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,64,128,1,float16,fp8,0,2.051365375518799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,64,0,1,fp8,fp8,0,2.7696107228597007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,64,128,1,fp8,fp8,0,2.6809972127278647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,64,0,1,float16,float16,0,3.014026641845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,64,128,1,float16,float16,0,2.1173280080159507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,64,0,1,float16,fp8,0,2.9991305669148765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,64,0,1,fp8,fp8,0,2.760117212931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,64,128,1,float16,fp8,0,2.0773706436157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,64,128,1,float16,float16,0,1.2157493432362874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,64,128,1,fp8,fp8,0,2.686095873514811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,64,0,1,float16,float16,0,3.0500160853068032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,64,128,1,float16,fp8,0,1.1908213297526042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,64,0,1,float16,float16,0,1.6890506744384766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,64,0,1,fp8,fp8,0,2.771962801615397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,64,0,1,float16,fp8,0,3.0238987604777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,64,128,1,fp8,fp8,0,1.5002719561258953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,64,128,1,float16,float16,0,1.0540053049723308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,64,0,1,float16,fp8,0,1.645359992980957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,64,0,1,fp8,fp8,0,1.5609493255615234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,64,128,1,float16,fp8,0,0.9971466859181722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,64,0,1,float16,float16,0,1.4798132578531902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,64,128,1,fp8,fp8,0,1.2946133613586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,64,0,1,float16,fp8,0,1.4653226534525554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,64,128,1,float16,float16,0,1.0451839764912922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,64,0,1,fp8,fp8,0,1.3362773259480794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,64,128,1,float16,fp8,0,0.9992160002390543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,64,128,1,fp8,fp8,0,1.3042399883270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,64,0,1,float16,float16,0,1.4656000137329102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,64,128,1,float16,float16,0,1.023098627726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,64,0,1,fp8,fp8,0,1.3524373372395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,64,0,1,float16,fp8,0,1.5221707026163738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,64,128,1,float16,fp8,0,1.0002826849619548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,64,0,1,float16,float16,0,1.5146239598592122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,64,128,1,fp8,fp8,0,1.3104533354441326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,64,128,1,float16,float16,0,1.035429318745931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,64,0,1,float16,fp8,0,1.46396271387736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,64,0,1,fp8,fp8,0,1.3951093355814617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,64,128,1,float16,fp8,0,1.0471093654632568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,64,0,1,float16,float16,0,1.493125279744466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,64,128,1,fp8,fp8,0,1.318837324778239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,64,128,1,float16,float16,0,0.6164586544036865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,64,128,1,float16,fp8,0,0.6105440060297648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,64,0,1,float16,float16,0,0.8466347058614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,64,0,1,fp8,fp8,0,1.3713547388712566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,64,0,1,float16,fp8,0,1.5142453511555989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,64,128,1,fp8,fp8,0,0.7576800187428793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,64,0,1,float16,fp8,0,0.8354026476542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,64,128,1,float16,float16,0,0.5273866653442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,64,0,1,fp8,fp8,0,0.7658507029215494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,64,128,1,float16,fp8,0,0.5180373191833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,64,0,1,float16,float16,0,0.7792267004648844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,64,128,1,fp8,fp8,0,0.6749333540598551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,64,0,1,float16,fp8,0,0.7579360008239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,64,0,1,fp8,fp8,0,0.7053279876708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,64,128,1,float16,float16,0,0.5300319989522299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,64,128,1,float16,fp8,0,0.5155200163523356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,64,0,1,float16,float16,0,0.7651413281758627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,64,128,1,fp8,fp8,0,0.6964693069458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,64,0,1,float16,fp8,0,0.7720692952473959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,64,0,1,fp8,fp8,0,0.6988586584726969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,64,128,1,float16,float16,0,0.5346986850102743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,64,128,1,float16,fp8,0,0.5228960116704305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,64,0,1,float16,float16,0,0.7706080277760824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,64,128,1,fp8,fp8,0,0.678938627243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,64,0,1,float16,fp8,0,0.761082649230957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,64,0,1,fp8,fp8,0,0.7205920219421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,64,128,1,float16,float16,0,0.5403253237406412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,64,128,1,float16,fp8,0,0.5336320002873739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,64,0,1,float16,float16,0,0.764789342880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,64,128,1,float16,float16,0,0.3238933285077413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,64,128,1,fp8,fp8,0,0.691093365351359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,64,128,1,float16,fp8,0,0.3210879961649577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,64,0,1,fp8,fp8,0,0.703770637512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,64,0,1,float16,fp8,0,0.7827253341674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,64,0,1,float16,float16,0,0.44811201095581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,64,128,1,fp8,fp8,0,0.407477339108785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,64,0,1,float16,fp8,0,0.4353119929631551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,64,0,1,fp8,fp8,0,0.4071893294652303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,64,128,1,float16,float16,0,0.28435732920964557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,64,128,1,float16,fp8,0,0.2882453401883443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,64,0,1,float16,float16,0,0.4012960195541382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,64,128,1,fp8,fp8,0,0.3703999916712443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,64,0,1,float16,fp8,0,0.4031360149383545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,64,0,1,fp8,fp8,0,0.3609813451766968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,64,128,1,float16,float16,0,0.28782399495442706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,64,0,1,float16,float16,0,0.4048853317896525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,64,128,1,float16,fp8,0,0.2893173297246297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,64,128,1,fp8,fp8,0,0.3675680160522461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,64,0,1,float16,fp8,0,0.4045120080312093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,64,0,1,fp8,fp8,0,0.3685866594314575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,64,128,1,float16,float16,0,0.2916799982388814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,64,128,1,fp8,fp8,0,0.38043200969696045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,64,128,1,float16,fp8,0,0.28384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,64,128,1,float16,float16,0,0.29155733187993366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,64,0,1,float16,float16,0,0.40968000888824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,64,0,1,float16,fp8,0,0.40323734283447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,64,0,1,fp8,fp8,0,0.37115732828776044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,64,0,1,float16,float16,0,0.4171733458836873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,64,128,1,float16,fp8,0,0.2933173378308614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,64,128,1,fp8,fp8,0,0.3773813247680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,64,0,1,float16,fp8,0,0.4095146656036377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,64,128,1,float16,float16,0,0.17930134137471518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,64,0,1,fp8,fp8,0,0.3718666632970174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,64,0,1,float16,float16,0,0.24662933746973673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,64,128,1,float16,fp8,0,0.18292800585428873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,64,0,1,float16,float16,0,0.21562665700912476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,64,128,1,fp8,fp8,0,0.23552000522613525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,64,0,1,float16,fp8,0,0.2451039950052897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,64,0,1,fp8,fp8,0,0.22267733017603555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,64,128,1,float16,float16,0,0.16332800189654031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,64,128,1,float16,fp8,0,0.1639840006828308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,64,128,1,fp8,fp8,0,0.21714133024215698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,64,0,1,float16,fp8,0,0.216538667678833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,64,0,1,fp8,fp8,0,0.2047626574834188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,64,128,1,float16,float16,0,0.167738676071167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,64,0,1,float16,float16,0,0.21613333622614542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,64,128,1,float16,fp8,0,0.16377066572507223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,64,128,1,fp8,fp8,0,0.21582400798797607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,64,0,1,float16,fp8,0,0.21573867400487265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,64,0,1,fp8,fp8,0,0.20544532934824625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,64,128,1,float16,float16,0,0.16808533668518066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,64,0,1,float16,float16,0,0.22048532962799072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,64,128,1,float16,fp8,0,0.16452800234158835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,64,128,1,fp8,fp8,0,0.21530133485794067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,64,0,1,float16,fp8,0,0.2158506711324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,64,0,1,fp8,fp8,0,0.20735466480255127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,64,128,1,float16,float16,0,0.16981865962346396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,64,0,1,float16,float16,0,0.22029866774876913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,64,128,1,float16,fp8,0,0.16884267330169678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,64,128,1,fp8,fp8,0,0.21862934033075967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,64,0,1,float16,fp8,0,0.21704532702763876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,64,128,1,float16,float16,0,0.12633599837621054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,64,0,1,fp8,fp8,0,0.2090453306833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,64,0,1,float16,float16,0,0.14867732922236124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,64,128,1,float16,fp8,0,0.12567466497421265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,64,128,1,fp8,fp8,0,0.1564586659272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,64,0,1,float16,fp8,0,0.14898666739463806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,64,0,1,fp8,fp8,0,0.1399999956289927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,64,128,1,float16,float16,0,0.12102400263150533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,64,0,1,float16,float16,0,0.14486933747927347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,64,128,1,float16,fp8,0,0.12107200423876445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,64,128,1,fp8,fp8,0,0.14320000012715658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,64,0,1,float16,fp8,0,0.14469333489735922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,64,0,1,fp8,fp8,0,0.138154665629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,64,128,1,float16,float16,0,0.12111999591191609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,64,0,1,float16,float16,0,0.14481066664059958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,64,128,1,float16,fp8,0,0.12132799625396729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,64,0,1,float16,fp8,0,0.14457600315411887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,64,128,1,fp8,fp8,0,0.1453279952208201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,64,0,1,fp8,fp8,0,0.1378506620724996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,64,128,1,float16,float16,0,0.1251306633154551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,64,0,1,fp8,fp8,0,0.1381226678689321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,64,0,1,float16,float16,0,0.14587199687957764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,64,128,1,float16,fp8,0,0.12245333194732666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,64,128,1,fp8,fp8,0,0.14915733536084494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,64,0,1,float16,fp8,0,0.14437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,64,128,1,float16,float16,0,0.12306666374206543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,64,0,1,float16,float16,0,0.14575466513633728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,64,128,1,float16,fp8,0,0.12380799651145935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,64,128,1,fp8,fp8,0,0.14707733194033304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,64,0,1,float16,fp8,0,0.14657066265741983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,64,0,1,fp8,fp8,0,0.13911466797192892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,64,128,1,float16,float16,0,3.104501406351725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,64,128,1,float16,fp8,0,3.07042662302653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,64,0,1,float16,float16,0,3.9711891810099282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,64,128,1,float16,float16,0,3.076784133911133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,64,0,1,fp8,fp8,0,3.4913225173950195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,64,128,1,fp8,fp8,0,3.8306347529093423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,64,0,1,float16,fp8,0,3.9210774103800454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,64,0,1,float16,float16,0,3.9304374059041343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,64,128,1,float16,fp8,0,3.04203192392985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,64,128,1,fp8,fp8,0,3.8667147954305015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,64,128,1,float16,float16,0,3.09877872467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,64,0,1,fp8,fp8,0,3.535642623901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,64,128,1,float16,fp8,0,3.0452426274617515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,64,0,1,float16,fp8,0,3.9045705795288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,64,0,1,float16,float16,0,3.9569546381632485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,64,128,1,fp8,fp8,0,3.8793598810831704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,64,0,1,float16,fp8,0,3.915701230367025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,64,128,1,float16,float16,0,3.1576054890950522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,64,0,1,fp8,fp8,0,3.525893211364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,64,128,1,float16,fp8,0,3.11247984568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,64,0,1,float16,float16,0,4.091621398925781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,64,128,1,fp8,fp8,0,3.97544002532959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,64,0,1,float16,fp8,0,3.9786879221598306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,64,128,1,float16,float16,0,1.8121013641357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,64,128,1,float16,fp8,0,1.7754240036010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,64,0,1,float16,float16,0,2.2302026748657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,64,0,1,fp8,fp8,0,3.6259358723958335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,64,128,1,fp8,fp8,0,2.2976853052775064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,64,0,1,float16,fp8,0,2.251525402069092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,64,128,1,float16,float16,0,1.4855413436889648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,64,128,1,float16,fp8,0,1.4548160235087078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,64,0,1,float16,float16,0,1.9200746218363445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,64,0,1,fp8,fp8,0,2.0696053504943848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,64,128,1,fp8,fp8,0,1.891642729441325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,64,128,1,float16,float16,0,1.5088319778442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,64,0,1,fp8,fp8,0,1.7639306386311848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,64,0,1,float16,fp8,0,1.9058292706807454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,64,128,1,float16,fp8,0,1.4936960538228352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,64,0,1,float16,float16,0,1.9194560050964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,64,128,1,fp8,fp8,0,1.924506664276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,64,0,1,float16,fp8,0,1.9089706738789876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,64,128,1,float16,float16,0,1.5324479738871257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,64,0,1,fp8,fp8,0,1.755344072977702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,64,128,1,float16,fp8,0,1.5176587104797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,64,0,1,float16,float16,0,1.9585973421732585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,64,128,1,fp8,fp8,0,1.9404212633768718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,64,0,1,float16,fp8,0,1.9206132888793945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,64,0,1,fp8,fp8,0,1.7663520177205403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,64,128,1,float16,float16,0,1.539365291595459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,64,128,1,float16,fp8,0,1.5290613174438477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,64,0,1,float16,float16,0,1.9749600092569988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,64,128,1,float16,float16,0,0.9057173728942871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,64,0,1,float16,float16,0,1.12227201461792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,64,128,1,fp8,fp8,0,1.9874505996704102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,64,0,1,float16,fp8,0,1.9672533671061199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,64,0,1,fp8,fp8,0,1.7872319221496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,64,128,1,float16,fp8,0,0.8784106572469076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,64,128,1,fp8,fp8,0,1.1004266738891602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,64,128,1,float16,float16,0,0.7547146479288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,64,0,1,fp8,fp8,0,1.0036426385243733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,64,0,1,float16,fp8,0,1.1030293305714924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,64,128,1,float16,fp8,0,0.7458293437957764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,64,0,1,float16,float16,0,0.9741866588592529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,64,128,1,fp8,fp8,0,0.9682986736297607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,64,0,1,float16,fp8,0,0.9647093613942465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,64,0,1,fp8,fp8,0,0.88155198097229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,64,128,1,float16,float16,0,0.7588586807250977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,64,128,1,float16,fp8,0,0.7536213397979736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,64,0,1,float16,float16,0,0.9736800193786621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,64,128,1,fp8,fp8,0,0.9752693176269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,64,0,1,float16,fp8,0,0.9694826602935791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,64,128,1,float16,float16,0,0.7688799699147543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,64,0,1,fp8,fp8,0,0.8834559917449951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,64,0,1,float16,float16,0,0.9797013600667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,64,128,1,float16,fp8,0,0.752778689066569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,64,128,1,fp8,fp8,0,0.9896533489227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,64,0,1,float16,fp8,0,0.9680319627126058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,64,0,1,fp8,fp8,0,0.8911466598510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,64,128,1,float16,float16,0,0.7756906350453695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,64,128,1,float16,fp8,0,0.7603840033213297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,64,0,1,float16,float16,0,0.991546630859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,64,128,1,fp8,fp8,0,1.0160533587137859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,64,128,1,float16,float16,0,0.4639093478520711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,64,128,1,float16,fp8,0,0.4572853247324626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,64,0,1,float16,fp8,0,0.9785706996917725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,64,0,1,float16,float16,0,0.573909322420756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,64,0,1,fp8,fp8,0,0.9069493611653646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,64,128,1,fp8,fp8,0,0.5656853516896566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,64,0,1,float16,fp8,0,0.5633226633071899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,64,0,1,fp8,fp8,0,0.5294186671574911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,64,128,1,float16,float16,0,0.39558398723602295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,64,128,1,float16,fp8,0,0.39077866077423096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,64,0,1,float16,float16,0,0.5086880127588908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,64,128,1,fp8,fp8,0,0.5119946797688802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,64,0,1,float16,fp8,0,0.5020639896392822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,64,0,1,fp8,fp8,0,0.4599999984105428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,64,128,1,float16,float16,0,0.3985813458760579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,64,0,1,float16,float16,0,0.5202026764551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,64,128,1,float16,fp8,0,0.39291199048360187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,64,128,1,fp8,fp8,0,0.5146453380584717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,64,0,1,float16,fp8,0,0.5061546564102173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,64,0,1,fp8,fp8,0,0.464789350827535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,64,128,1,float16,float16,0,0.4028746684392293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,64,0,1,float16,float16,0,0.5081119934717814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,64,128,1,float16,fp8,0,0.3951946496963501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,64,0,1,fp8,fp8,0,0.46692268053690594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,64,128,1,fp8,fp8,0,0.5237386624018351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,64,0,1,float16,fp8,0,0.5040320158004761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,64,128,1,float16,float16,0,0.4060320059458415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,64,128,1,float16,fp8,0,0.4007893403371175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,64,0,1,float16,float16,0,0.5161706606547037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,64,128,1,fp8,fp8,0,0.5256479978561401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,64,0,1,fp8,fp8,0,0.47168533007303876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,64,0,1,float16,fp8,0,0.5115146636962891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,64,128,1,float16,float16,0,0.24992533524831137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,64,0,1,float16,float16,0,0.30399467547734577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,64,128,1,float16,fp8,0,0.24489066998163858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,64,128,1,fp8,fp8,0,0.31171200672785443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,64,0,1,float16,fp8,0,0.3009653290112813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,64,0,1,float16,float16,0,0.2768266598383586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,64,0,1,fp8,fp8,0,0.2747466762860616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,64,128,1,float16,float16,0,0.2174826661745707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,64,128,1,float16,fp8,0,0.21541333198547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,64,128,1,fp8,fp8,0,0.2837173342704773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,64,0,1,float16,fp8,0,0.2717653314272563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,64,0,1,float16,float16,0,0.2741439938545227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,64,0,1,fp8,fp8,0,0.24644800027211508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,64,128,1,float16,float16,0,0.222271998723348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,64,128,1,float16,fp8,0,0.21563732624053955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,64,128,1,fp8,fp8,0,0.2862880031267802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,64,0,1,float16,fp8,0,0.2720106641451518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,64,0,1,fp8,fp8,0,0.2537226676940918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,64,128,1,fp8,fp8,0,0.2874026695887248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,64,0,1,float16,fp8,0,0.2744266589482625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,64,0,1,fp8,fp8,0,0.2493226726849874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,64,128,1,float16,float16,0,0.219050665696462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,64,0,1,float16,float16,0,0.2767573396364848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,64,128,1,float16,fp8,0,0.2205386757850647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,64,128,1,float16,float16,0,0.22627200682957968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,64,128,1,float16,fp8,0,0.2207253376642863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,64,0,1,float16,float16,0,0.2800053358078003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,64,128,1,float16,float16,0,0.14286399881045023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,64,128,1,fp8,fp8,0,0.28902933994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,64,128,1,float16,fp8,0,0.14180800318717957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,64,0,1,float16,fp8,0,0.27799467245737713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,64,0,1,fp8,fp8,0,0.2512800097465515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,64,0,1,float16,float16,0,0.16765866676966348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,64,128,1,fp8,fp8,0,0.18232532342274985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,64,0,1,float16,float16,0,0.14857600132624307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,64,0,1,float16,fp8,0,0.16899200280507407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,64,0,1,fp8,fp8,0,0.15478932857513428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,64,128,1,float16,float16,0,0.12813867131868997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,64,128,1,float16,fp8,0,0.12824533383051553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,64,0,1,float16,float16,0,0.1488533318042755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,64,128,1,fp8,fp8,0,0.16925332943598428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,64,0,1,float16,fp8,0,0.14832533399264017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,64,0,1,fp8,fp8,0,0.1406880021095276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,64,128,1,float16,float16,0,0.12983466188112894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,64,128,1,float16,fp8,0,0.12823999921480814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,64,128,1,fp8,fp8,0,0.16814400752385458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,64,0,1,float16,float16,0,0.14938132961591086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,64,0,1,float16,fp8,0,0.14839999874432883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,64,0,1,fp8,fp8,0,0.14332800110181174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,64,128,1,float16,float16,0,0.12957866986592612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,64,128,1,float16,fp8,0,0.12916800379753113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,64,128,1,fp8,fp8,0,0.17070400714874268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,64,0,1,float16,fp8,0,0.14915733536084494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,64,0,1,fp8,fp8,0,0.14239999651908875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,64,128,1,float16,float16,0,0.130213330189387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,64,0,1,float16,float16,0,0.1523306667804718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,64,128,1,float16,fp8,0,0.12933866182963052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,64,128,1,fp8,fp8,0,0.17203199863433838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,64,0,1,float16,fp8,0,0.14966400464375815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,64,0,1,fp8,fp8,0,0.14491732915242514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,64,128,1,float16,float16,0,0.09963200489679973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,64,0,1,float16,float16,0,0.10780266920725505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,64,128,1,float16,fp8,0,0.10041067004203796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,64,128,1,fp8,fp8,0,0.1253653367360433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,64,0,1,float16,fp8,0,0.10838933785756429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,64,0,1,fp8,fp8,0,0.10252267122268677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,64,128,1,fp8,fp8,0,0.11410133043924968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,64,128,1,float16,float16,0,0.09458133578300476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,64,0,1,float16,float16,0,0.10481066505114238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,64,128,1,float16,fp8,0,0.09409600496292114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,64,0,1,float16,fp8,0,0.10499200224876404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,64,0,1,fp8,fp8,0,0.10026133060455322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,64,128,1,float16,float16,0,0.09373866518338521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,64,0,1,float16,float16,0,0.10488532980283101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,64,128,1,float16,fp8,0,0.09469333291053772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,64,128,1,fp8,fp8,0,0.1133013367652893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,64,0,1,float16,fp8,0,0.10443199674288432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,64,0,1,fp8,fp8,0,0.10050132870674133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,64,128,1,float16,float16,0,0.09459199508031209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,64,0,1,float16,float16,0,0.10517332951227824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,64,0,1,float16,float16,0,0.10493866602579753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,64,128,1,float16,fp8,0,0.09412800272305806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,64,128,1,float16,fp8,0,0.09505066275596619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,64,128,1,fp8,fp8,0,0.11531200011571248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,64,0,1,float16,fp8,0,0.10470400253931682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,64,0,1,fp8,fp8,0,0.10016533732414246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,64,128,1,float16,float16,0,0.09492266178131104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,64,128,1,fp8,fp8,0,0.1165120005607605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,64,0,1,float16,fp8,0,0.10527466734250386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,64,0,1,fp8,fp8,0,0.09962667028109233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,64,128,1,float16,float16,0,3.639258702596029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,64,128,1,float16,fp8,0,3.6386613845825195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,64,0,1,float16,float16,0,4.1932799021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,64,0,1,fp8,fp8,0,3.7330827713012695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,64,128,1,float16,float16,0,3.693632125854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,64,0,1,float16,fp8,0,4.188543955485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,64,128,1,fp8,fp8,0,4.765418688456218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,64,0,1,float16,float16,0,4.226629257202148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,64,128,1,float16,fp8,0,3.6727892557779946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,64,0,1,fp8,fp8,0,3.758869489034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,64,128,1,float16,float16,0,3.8072532018025718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,64,0,1,float16,fp8,0,4.144106547037761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,64,128,1,fp8,fp8,0,4.845829327901204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,64,128,1,float16,fp8,0,3.7677653630574546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,64,0,1,float16,float16,0,4.336031913757324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,64,128,1,float16,float16,0,3.7822399139404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,64,0,1,fp8,fp8,0,3.90340264638265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,64,128,1,fp8,fp8,0,4.912853240966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,64,0,1,float16,fp8,0,4.291130701700847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,64,128,1,float16,fp8,0,3.8376105626424155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,64,0,1,float16,float16,0,4.350629488627116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,64,128,1,fp8,fp8,0,4.984512011210124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,64,128,1,float16,float16,0,2.2104320526123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,64,128,1,float16,fp8,0,2.1514612833658853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,64,0,1,float16,float16,0,2.4618453979492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,64,128,1,fp8,fp8,0,2.597482681274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,64,0,1,float16,fp8,0,2.4398345947265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,64,0,1,fp8,fp8,0,4.001450538635254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,64,0,1,fp8,fp8,0,2.077125390370687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,64,0,1,float16,fp8,0,4.378138542175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,64,128,1,float16,float16,0,1.811029275258382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,64,128,1,float16,fp8,0,1.7861119906107585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,64,0,1,float16,float16,0,2.1276159286499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,64,0,1,float16,fp8,0,2.130400021870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,64,128,1,fp8,fp8,0,2.37065060933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,64,0,1,fp8,fp8,0,1.8587199846903484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,64,128,1,float16,float16,0,1.8153707186381023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,64,0,1,float16,float16,0,2.090223948160807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,64,128,1,float16,fp8,0,1.8606986999511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,64,0,1,float16,fp8,0,2.1537653605143228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,64,128,1,fp8,fp8,0,2.457376003265381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,64,128,1,float16,float16,0,1.866165320078532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,64,0,1,fp8,fp8,0,1.9550666809082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,64,128,1,float16,fp8,0,1.9149120648701985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,64,0,1,float16,float16,0,2.165605386098226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,64,0,1,float16,fp8,0,2.191813309987386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,64,128,1,fp8,fp8,0,2.555034637451172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,64,0,1,fp8,fp8,0,1.9757866859436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,64,128,1,float16,float16,0,1.9381386439005535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,64,0,1,float16,float16,0,2.210810661315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,64,128,1,float16,fp8,0,1.9577919642130535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,64,128,1,float16,float16,0,1.1242667039235432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,64,0,1,float16,fp8,0,2.1996800104777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,64,0,1,float16,float16,0,1.2482240200042725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,64,128,1,float16,fp8,0,1.0781546433766682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,64,128,1,fp8,fp8,0,2.5403946240743003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,64,0,1,fp8,fp8,0,2.01366392771403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,64,128,1,fp8,fp8,0,1.33187731107076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,64,0,1,float16,fp8,0,1.2292160193125408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,64,0,1,fp8,fp8,0,1.050165335337321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,64,128,1,float16,float16,0,0.9306666851043701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,64,0,1,float16,float16,0,1.0655146439870198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,64,128,1,float16,fp8,0,0.9240000247955322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,64,0,1,float16,fp8,0,1.066975990931193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,64,0,1,fp8,fp8,0,0.960319995880127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,64,0,1,float16,float16,0,1.0622933705647786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,64,128,1,fp8,fp8,0,1.231925328572591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,64,128,1,float16,float16,0,0.951200008392334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,64,128,1,float16,fp8,0,0.9371413389841715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,64,0,1,float16,fp8,0,1.0796319643656414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,64,128,1,fp8,fp8,0,1.2375306288401287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,64,0,1,fp8,fp8,0,0.9724853038787842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,64,128,1,float16,float16,0,0.9651892979939779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,64,0,1,float16,float16,0,1.0841386318206787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,64,128,1,float16,fp8,0,0.9505706628163656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,64,128,1,fp8,fp8,0,1.2807679971059163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,64,0,1,float16,fp8,0,1.093183994293213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,64,0,1,fp8,fp8,0,1.0052213668823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,64,128,1,float16,float16,0,0.9566026528676351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,64,0,1,float16,float16,0,1.0968159834543865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,64,128,1,float16,fp8,0,0.9633440176645914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,64,128,1,float16,float16,0,0.5679306586583456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,64,0,1,float16,fp8,0,1.0825226306915283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,64,128,1,fp8,fp8,0,1.2789493401845295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,64,0,1,float16,float16,0,0.6393973429997762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,64,128,1,float16,fp8,0,0.553061326344808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,64,0,1,fp8,fp8,0,1.016144037246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,64,0,1,float16,fp8,0,0.6224266688028971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,64,128,1,fp8,fp8,0,0.6844053268432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,64,0,1,fp8,fp8,0,0.5373173157374064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,64,128,1,float16,float16,0,0.4797653357187907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,64,0,1,float16,float16,0,0.5436426798502604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,64,128,1,float16,fp8,0,0.47890134652455646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,64,128,1,fp8,fp8,0,0.6267520189285278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,64,0,1,float16,fp8,0,0.5484426816304525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,64,0,1,fp8,fp8,0,0.4894026517868042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,64,128,1,float16,float16,0,0.4817013343175252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,64,128,1,float16,fp8,0,0.4816746711730957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,64,0,1,float16,float16,0,0.5480426549911499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,64,0,1,float16,fp8,0,0.553706685702006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,64,128,1,float16,float16,0,0.4875093301137288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,64,128,1,fp8,fp8,0,0.6315413316090902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,64,0,1,fp8,fp8,0,0.49375466505686444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,64,0,1,float16,float16,0,0.5604639848073324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,64,128,1,fp8,fp8,0,0.6409600178400675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,64,128,1,float16,fp8,0,0.4928319851557414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,64,0,1,float16,fp8,0,0.5556000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,64,0,1,fp8,fp8,0,0.5032320022583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,64,128,1,float16,float16,0,0.48761598269144696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,64,128,1,float16,fp8,0,0.48953068256378174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,64,0,1,float16,float16,0,0.5563466548919678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,64,128,1,fp8,fp8,0,0.6453760067621866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,64,0,1,float16,fp8,0,0.5562026500701904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,64,128,1,float16,float16,0,0.29708800713221234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,64,0,1,fp8,fp8,0,0.5059786637624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,64,0,1,float16,float16,0,0.32944534222284955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,64,128,1,float16,fp8,0,0.2893279989560445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,64,128,1,fp8,fp8,0,0.35789334774017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,64,0,1,float16,fp8,0,0.32391466697057086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,64,0,1,fp8,fp8,0,0.2753653327624003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,64,128,1,float16,float16,0,0.2524320085843404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,64,0,1,float16,float16,0,0.2856160004933675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,64,128,1,float16,fp8,0,0.25147199630737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,64,128,1,fp8,fp8,0,0.3321546713511149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,64,0,1,float16,fp8,0,0.28648533423741657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,64,0,1,fp8,fp8,0,0.24871466557184854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,64,128,1,float16,float16,0,0.25224000215530396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,64,0,1,float16,float16,0,0.28703999519348145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,64,128,1,float16,fp8,0,0.2521493236223857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,64,128,1,fp8,fp8,0,0.33452268441518146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,64,0,1,fp8,fp8,0,0.25381867090861004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,64,128,1,fp8,fp8,0,0.3378186623255412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,64,0,1,float16,fp8,0,0.28752533594767254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,64,128,1,float16,float16,0,0.2570506731669108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,64,0,1,float16,float16,0,0.29197333256403607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,64,128,1,float16,fp8,0,0.2577173312505086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,64,0,1,float16,fp8,0,0.29152532418568927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,64,0,1,fp8,fp8,0,0.25761600335439044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,64,128,1,float16,float16,0,0.2568639914194743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,64,0,1,float16,float16,0,0.2940959930419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,64,0,1,fp8,fp8,0,0.256661335627238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,64,128,1,float16,fp8,0,0.259061336517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,64,128,1,fp8,fp8,0,0.33779199918111164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,64,0,1,float16,fp8,0,0.2939680020014445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,64,128,1,float16,float16,0,0.16234667102495828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,64,0,1,float16,float16,0,0.18448533614476523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,64,128,1,float16,fp8,0,0.1590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,64,128,1,fp8,fp8,0,0.19917333126068115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,64,0,1,float16,fp8,0,0.1821333368619283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,64,0,1,fp8,fp8,0,0.14689067006111145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,64,128,1,float16,float16,0,0.13638933499654135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,64,0,1,float16,float16,0,0.15057067076365152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,64,128,1,float16,fp8,0,0.13743467132250467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,64,128,1,float16,float16,0,0.1373546620210012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,64,128,1,fp8,fp8,0,0.1825066606203715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,64,0,1,float16,fp8,0,0.14908799529075623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,64,0,1,fp8,fp8,0,0.1356000006198883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,64,0,1,float16,float16,0,0.14973333477973938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,64,128,1,float16,fp8,0,0.13875733812650046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,64,128,1,fp8,fp8,0,0.18291733662287393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,64,0,1,float16,fp8,0,0.14966932932535806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,64,0,1,fp8,fp8,0,0.13450666268666586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,64,128,1,float16,float16,0,0.14096533258756003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,64,0,1,float16,float16,0,0.15229866902033487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,64,128,1,float16,fp8,0,0.14006400108337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,64,128,1,fp8,fp8,0,0.18746666113535562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,64,0,1,float16,fp8,0,0.15095999836921692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,64,0,1,fp8,fp8,0,0.13806399703025818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,64,128,1,float16,float16,0,0.14314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,64,0,1,float16,float16,0,0.15388799707094827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,64,128,1,float16,fp8,0,0.14241600036621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,64,128,1,fp8,fp8,0,0.1872426668802897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,64,0,1,float16,fp8,0,0.15288533767064413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,64,0,1,fp8,fp8,0,0.14010666807492575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,64,128,1,float16,float16,0,0.09298666318257649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,64,0,1,float16,float16,0,0.09482133388519287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,64,128,1,float16,fp8,0,0.09120532870292664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,64,128,1,fp8,fp8,0,0.11821333567301433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,64,0,1,float16,fp8,0,0.09475200374921162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,64,0,1,fp8,fp8,0,0.08556800087292989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,64,128,1,float16,float16,0,0.07773866752783458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,64,0,1,float16,float16,0,0.08313600222269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,64,128,1,float16,fp8,0,0.07820799946784973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,64,128,1,fp8,fp8,0,0.10255466898282369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,64,0,1,float16,fp8,0,0.08343467116355896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,64,0,1,fp8,fp8,0,0.07526933153470357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,64,128,1,float16,float16,0,0.07919466495513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,64,0,1,float16,float16,0,0.08410666386286418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,64,0,1,float16,float16,0,0.08506666620572408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,64,128,1,float16,fp8,0,0.0791733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,64,128,1,fp8,fp8,0,0.10285866260528564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,64,0,1,float16,fp8,0,0.08341866731643677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,64,0,1,fp8,fp8,0,0.07684800028800964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,64,128,1,float16,float16,0,0.08165866633256276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,64,128,1,float16,fp8,0,0.08213866750399272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,64,128,1,fp8,fp8,0,0.10687466462453206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,64,0,1,float16,fp8,0,0.0857973297437032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,64,0,1,fp8,fp8,0,0.07830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,64,128,1,float16,float16,0,0.08227733274300893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,64,0,1,float16,float16,0,0.08646933237711589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,64,128,1,float16,fp8,0,0.08292800188064575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,64,128,1,float16,fp8,0,0.052799999713897705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,64,128,1,fp8,fp8,0,0.11010133226712544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,64,0,1,float16,fp8,0,0.08728532989819844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,64,0,1,fp8,fp8,0,0.07898133496443431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,64,128,1,float16,float16,0,0.05560533205668131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,64,0,1,float16,float16,0,0.054885332783063255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,64,128,1,fp8,fp8,0,0.07208000123500824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,64,0,1,float16,fp8,0,0.05551466842492422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,64,0,1,fp8,fp8,0,0.0507893313964208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,64,128,1,float16,float16,0,0.04833066463470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,64,0,1,float16,float16,0,0.050293331344922386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,64,128,1,float16,fp8,0,0.04867733518282572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,64,128,1,fp8,fp8,0,0.06527466575304668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,64,0,1,float16,fp8,0,0.05012799799442291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,64,0,1,fp8,fp8,0,0.04686399797598521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,64,128,1,float16,float16,0,0.04955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,64,0,1,float16,float16,0,0.05055999755859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,64,128,1,float16,fp8,0,0.049440001447995506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,64,0,1,float16,float16,0,0.05201066533724467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,64,128,1,fp8,fp8,0,0.06284266710281372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,64,0,1,float16,fp8,0,0.051226665576299034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,64,0,1,fp8,fp8,0,0.04660800099372864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,64,128,1,float16,float16,0,0.05009600023428599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,64,128,1,float16,fp8,0,0.05000533163547516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,64,128,1,fp8,fp8,0,0.06541333099206288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,64,0,1,float16,fp8,0,0.051967998345692955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,64,0,1,fp8,fp8,0,0.04844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,64,128,1,float16,float16,0,0.0506879985332489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,64,0,1,float16,float16,0,0.051551997661590576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,64,128,1,float16,fp8,0,0.05050133168697357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,64,128,1,fp8,fp8,0,0.06343466540177663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,64,0,1,float16,fp8,0,0.05207466582457224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,64,0,1,fp8,fp8,0,0.04795733094215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,64,128,1,float16,float16,0,3.437525431315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,64,0,1,float16,float16,0,3.4471041361490884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,64,128,1,float16,fp8,0,3.4398934046427407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,64,0,1,fp8,fp8,0,3.0313332875569663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,64,0,1,float16,fp8,0,3.423114776611328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,64,128,1,float16,float16,0,3.505839983622233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,64,128,1,fp8,fp8,0,4.495290756225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,64,0,1,float16,float16,0,3.4420639673868814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,64,128,1,float16,fp8,0,3.4793227513631186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,64,0,1,fp8,fp8,0,3.05348269144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,64,0,1,float16,fp8,0,3.425130526224772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,64,128,1,float16,float16,0,3.634415944417318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,64,128,1,fp8,fp8,0,4.531290690104167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,64,0,1,float16,float16,0,3.5818347930908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,64,128,1,float16,fp8,0,3.544831911722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,64,0,1,fp8,fp8,0,3.1653919219970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,64,0,1,float16,fp8,0,3.490031878153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,64,128,1,float16,float16,0,3.6214987436930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,64,128,1,fp8,fp8,0,4.598069190979004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,64,0,1,float16,float16,0,3.631535847981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,64,128,1,float16,fp8,0,3.631797472635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,64,0,1,float16,fp8,0,3.585850715637207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,64,128,1,fp8,fp8,0,4.682858784993489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,64,128,1,float16,float16,0,2.099642594655355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,64,0,1,float16,float16,0,2.1122825940450034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,64,128,1,float16,fp8,0,2.028261343638102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,64,0,1,fp8,fp8,0,3.2192214330037436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,64,0,1,float16,fp8,0,2.0244906743367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,64,128,1,fp8,fp8,0,2.476736068725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,64,0,1,fp8,fp8,0,1.709162712097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,64,0,1,float16,float16,0,1.6757173538208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,64,128,1,float16,float16,0,1.6911253929138184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,64,128,1,float16,fp8,0,1.7496053377787273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,64,0,1,float16,fp8,0,1.6939627329508464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,64,0,1,fp8,fp8,0,1.4855945905049641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,64,128,1,fp8,fp8,0,2.2163413365681968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,64,0,1,float16,float16,0,1.693984031677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,64,128,1,float16,float16,0,1.718821366628011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,64,128,1,float16,fp8,0,1.7078612645467122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,64,0,1,float16,fp8,0,1.7231626510620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,64,0,1,fp8,fp8,0,1.5617653528849285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,64,128,1,fp8,fp8,0,2.24509859085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,64,128,1,float16,float16,0,1.779520034790039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,64,0,1,float16,float16,0,1.7636747360229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,64,128,1,float16,fp8,0,1.763162612915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,64,0,1,float16,fp8,0,1.7493707338968914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,64,128,1,fp8,fp8,0,2.314880053202311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,64,0,1,fp8,fp8,0,1.5838720003763835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,64,128,1,float16,float16,0,1.8464800516764324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,64,0,1,float16,float16,0,1.769327958424886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,64,128,1,float16,fp8,0,1.7880746523539226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,64,128,1,float16,float16,0,1.06112535794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,64,0,1,float16,float16,0,1.0573279857635498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,64,128,1,fp8,fp8,0,2.3287787437438965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,64,0,1,float16,fp8,0,1.7673333485921223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,64,0,1,fp8,fp8,0,1.5968586603800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,64,128,1,float16,fp8,0,1.044042666753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,64,128,1,fp8,fp8,0,1.2340853214263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,64,0,1,float16,fp8,0,1.0108959674835205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,64,0,1,fp8,fp8,0,0.8766986529032389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,64,128,1,float16,float16,0,0.8877813021341959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,64,0,1,float16,float16,0,0.8470133145650228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,64,128,1,float16,fp8,0,0.8591146469116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,64,0,1,float16,fp8,0,0.8756799697875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,64,0,1,fp8,fp8,0,0.7482559680938721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,64,128,1,fp8,fp8,0,1.1258933544158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,64,128,1,float16,float16,0,0.8693280220031738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,64,0,1,float16,float16,0,0.8849973678588867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,64,128,1,float16,fp8,0,0.8986506462097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,64,0,1,float16,fp8,0,0.8509759902954102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,64,128,1,fp8,fp8,0,1.1677013238271077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,64,0,1,fp8,fp8,0,0.7870079676310221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,64,128,1,float16,float16,0,0.8920959631601969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,64,0,1,float16,float16,0,0.8776319821675619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,64,128,1,float16,fp8,0,0.8898879686991373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,64,0,1,float16,fp8,0,0.9076853593190511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,64,0,1,fp8,fp8,0,0.7858453591664633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,64,128,1,fp8,fp8,0,1.202672004699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,64,128,1,float16,float16,0,0.9152426719665527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,64,0,1,float16,float16,0,0.8744640350341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,64,128,1,float16,fp8,0,0.8862026532491049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,64,128,1,float16,float16,0,0.5444533427556356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,64,0,1,float16,float16,0,0.5383306741714478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,64,0,1,float16,fp8,0,0.9051626523335775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,64,0,1,fp8,fp8,0,0.7996213436126709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,64,128,1,fp8,fp8,0,1.1779413223266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,64,128,1,float16,fp8,0,0.5203306674957275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,64,128,1,fp8,fp8,0,0.6344480117162069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,64,0,1,fp8,fp8,0,0.4471520185470581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,64,0,1,float16,fp8,0,0.5230933427810669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,64,128,1,float16,float16,0,0.45774932702382404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,64,0,1,float16,float16,0,0.43346134821573895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,64,128,1,float16,fp8,0,0.44276265303293866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,64,128,1,fp8,fp8,0,0.5775839885075887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,64,0,1,fp8,fp8,0,0.39555732409159344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,64,0,1,float16,fp8,0,0.4331626494725545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,64,128,1,float16,float16,0,0.4586293299992879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,64,0,1,float16,float16,0,0.4503786563873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,64,128,1,float16,fp8,0,0.44651734828948975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,64,0,1,fp8,fp8,0,0.4037439823150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,64,128,1,fp8,fp8,0,0.5830293496449789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,64,0,1,float16,fp8,0,0.45046401023864746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,64,128,1,float16,float16,0,0.4562026659647624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,64,0,1,float16,float16,0,0.4562346537907918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,64,128,1,float16,fp8,0,0.47045334180196124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,64,128,1,fp8,fp8,0,0.6014666557312012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,64,0,1,float16,fp8,0,0.4599573214848836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,64,0,1,fp8,fp8,0,0.39663998285929364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,64,128,1,float16,float16,0,0.468666672706604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,64,0,1,float16,float16,0,0.44768532117207843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,64,128,1,float16,fp8,0,0.465770681699117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,64,128,1,fp8,fp8,0,0.6143680016199747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,64,0,1,float16,fp8,0,0.4631839990615845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,64,0,1,fp8,fp8,0,0.4126773277918498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,64,128,1,float16,float16,0,0.28199466069539386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,64,128,1,float16,fp8,0,0.2822879950205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,64,0,1,float16,float16,0,0.2794293363889058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,64,128,1,fp8,fp8,0,0.33647998174031574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,64,0,1,float16,fp8,0,0.2716853419939677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,64,0,1,fp8,fp8,0,0.23116799195607504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,64,128,1,float16,float16,0,0.23868266741434732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,64,0,1,float16,float16,0,0.22978132963180542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,64,128,1,float16,fp8,0,0.24227199951807657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,64,128,1,fp8,fp8,0,0.3163893421490987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,64,0,1,float16,fp8,0,0.23411200443903604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,64,0,1,fp8,fp8,0,0.2055093248685201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,64,128,1,float16,float16,0,0.2428426742553711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,64,0,1,float16,float16,0,0.23478400707244873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,64,128,1,float16,fp8,0,0.24285332361857095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,64,128,1,fp8,fp8,0,0.3182826638221741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,64,0,1,float16,fp8,0,0.23491734266281128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,64,0,1,fp8,fp8,0,0.20759467283884683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,64,128,1,float16,float16,0,0.24680533011754355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,64,0,1,float16,float16,0,0.24015466372172037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,64,128,1,float16,fp8,0,0.2466986576716105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,64,128,1,fp8,fp8,0,0.32344533999760944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,64,0,1,float16,fp8,0,0.2439039945602417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,64,0,1,fp8,fp8,0,0.21229867140452066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,64,128,1,float16,float16,0,0.24683199326197305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,64,0,1,float16,float16,0,0.24009066820144653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,64,128,1,float16,fp8,0,0.24715733528137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,64,128,1,fp8,fp8,0,0.3187519907951355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,64,0,1,float16,fp8,0,0.24043200413386026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,64,0,1,fp8,fp8,0,0.2145813306172689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,64,128,1,float16,float16,0,0.15569600462913513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,64,0,1,float16,float16,0,0.1644533375898997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,64,128,1,float16,fp8,0,0.15307733416557312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,64,128,1,fp8,fp8,0,0.18825066089630127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,64,0,1,float16,fp8,0,0.15728533267974854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,64,0,1,fp8,fp8,0,0.12468799948692322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,64,128,1,float16,float16,0,0.13120533029238382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,64,0,1,float16,float16,0,0.12598933776219687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,64,128,1,float16,fp8,0,0.1322933336098989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,64,128,1,fp8,fp8,0,0.17469332615534464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,64,0,1,float16,fp8,0,0.12657599647839865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,64,0,1,fp8,fp8,0,0.11214933792750041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,64,128,1,float16,float16,0,0.13260799646377563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,64,0,1,float16,float16,0,0.12668266892433167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,64,128,1,float16,fp8,0,0.13194132844607034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,64,128,1,fp8,fp8,0,0.1737333337465922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,64,0,1,float16,fp8,0,0.12574399511019388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,64,0,1,fp8,fp8,0,0.1142080028851827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,64,128,1,float16,float16,0,0.13540800412495932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,64,0,1,float16,float16,0,0.12821867068608603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,64,128,1,float16,fp8,0,0.13407466808954874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,64,128,1,fp8,fp8,0,0.1776533325513204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,64,0,1,float16,fp8,0,0.1277653376261393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,64,0,1,fp8,fp8,0,0.1156160036722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,64,128,1,float16,float16,0,0.1363040010134379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,64,0,1,float16,float16,0,0.1304853359858195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,64,128,1,float16,fp8,0,0.136954665184021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,64,128,1,fp8,fp8,0,0.17709867159525552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,64,0,1,float16,fp8,0,0.13024000326792398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,64,0,1,fp8,fp8,0,0.11716799934705098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,64,128,1,float16,float16,0,0.09040000041325887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,64,0,1,float16,float16,0,0.08402666449546814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,64,128,1,float16,fp8,0,0.08918933073679607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,64,128,1,fp8,fp8,0,0.11141866445541382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,64,0,1,float16,fp8,0,0.08272533118724823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,64,0,1,fp8,fp8,0,0.07427200178305308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,64,128,1,float16,float16,0,0.07531733314196269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,64,0,1,float16,float16,0,0.0705973356962204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,64,128,1,float16,fp8,0,0.07587733368078868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,64,128,1,fp8,fp8,0,0.09667199850082397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,64,0,1,float16,fp8,0,0.07120533287525177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,64,0,1,fp8,fp8,0,0.06394133468468984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,64,128,1,float16,float16,0,0.0774239997069041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,64,0,1,float16,float16,0,0.07177066802978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,64,128,1,float16,fp8,0,0.07610666751861572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,64,128,1,fp8,fp8,0,0.09950400392214458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,64,0,1,float16,fp8,0,0.0727893312772115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,64,128,1,fp8,fp8,0,0.10097600022951762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,64,0,1,fp8,fp8,0,0.06494933366775513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,64,128,1,float16,float16,0,0.07836266855398814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,64,0,1,float16,float16,0,0.07340266803900401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,64,128,1,float16,fp8,0,0.07807466884454091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,64,0,1,float16,fp8,0,0.07434133191903432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,64,0,1,fp8,fp8,0,0.06762666503588359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,64,128,1,float16,float16,0,0.07933866480986278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,64,0,1,float16,float16,0,0.07574399809042613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,64,128,1,float16,fp8,0,0.07973333199818929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,64,128,1,fp8,fp8,0,0.10236799716949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,64,0,1,float16,fp8,0,0.07578666508197784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,64,0,1,fp8,fp8,0,0.06690666576226552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,64,128,1,float16,float16,0,0.05429333448410034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,64,0,1,float16,float16,0,0.04868799944718679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,64,128,1,float16,fp8,0,0.05340266724427541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,64,128,1,fp8,fp8,0,0.06714133421579997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,64,0,1,float16,fp8,0,0.0470719983180364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,64,128,1,fp8,fp8,0,0.06166933476924896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,64,0,1,fp8,fp8,0,0.04348800083001455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,64,128,1,float16,float16,0,0.04713066418965658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,64,0,1,float16,float16,0,0.044112001856168113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,64,0,1,float16,float16,0,0.043968002001444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,64,128,1,float16,fp8,0,0.04834666848182678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,64,0,1,float16,fp8,0,0.043706665436426796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,64,0,1,fp8,fp8,0,0.03997866561015447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,64,0,1,fp8,fp8,0,0.04045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,64,128,1,float16,float16,0,0.04858666658401489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,64,128,1,float16,fp8,0,0.049226666490236916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,64,128,1,fp8,fp8,0,0.0624533345301946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,64,0,1,float16,fp8,0,0.04418133199214935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,64,128,1,float16,float16,0,0.049039999643961586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,64,0,1,float16,float16,0,0.044906665881474815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,64,128,1,float16,fp8,0,0.04849599798520406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,64,128,1,fp8,fp8,0,0.06309333443641663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,64,0,1,float16,fp8,0,0.045168002446492515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,64,0,1,fp8,fp8,0,0.04147200038035711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,64,128,1,float16,float16,0,0.04959466556708018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,64,0,1,float16,float16,0,0.045381332437197365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,64,128,1,float16,fp8,0,0.05051200091838837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,64,128,1,fp8,fp8,0,0.06206933160622915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,64,0,1,float16,fp8,0,0.046336000164349876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,64,0,1,fp8,fp8,0,0.041375999649365745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,64,128,1,float16,float16,0,0.039936001102129616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,64,0,1,float16,float16,0,0.03431999931732813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,64,128,1,float16,fp8,0,0.03766400118668874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,64,128,1,fp8,fp8,0,0.04244266450405121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,64,128,1,float16,fp8,0,0.034629332522551216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,64,0,1,float16,fp8,0,0.034703999757766724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,64,0,1,fp8,fp8,0,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,64,128,1,float16,float16,0,0.03509866694609324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,64,0,1,float16,float16,0,0.031983998914559685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,64,128,1,fp8,fp8,0,0.04065600037574768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,64,128,1,fp8,fp8,0,0.041152000427246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,64,0,1,float16,fp8,0,0.032618666688601174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,64,0,1,fp8,fp8,0,0.030762667457262676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,64,128,1,float16,float16,0,0.03626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,64,0,1,float16,float16,0,0.032218667368094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,64,128,1,float16,fp8,0,0.03457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,64,0,1,float16,fp8,0,0.0322026660044988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,64,0,1,fp8,fp8,0,0.030085332691669464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,64,128,1,float16,float16,0,0.03533333291610082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,64,0,1,float16,float16,0,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,64,128,1,float16,fp8,0,0.03629866739114126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,64,128,1,fp8,fp8,0,0.04165333261092504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,64,128,1,fp8,fp8,0,0.04164266586303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,64,0,1,float16,fp8,0,0.033600000043710075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,64,0,1,float16,fp8,0,0.03313600023587545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,64,0,1,fp8,fp8,0,0.03025600065787633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,64,128,1,float16,float16,0,0.03762666632731756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,64,0,1,float16,float16,0,0.033301333586374916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,64,128,1,float16,fp8,0,0.03588266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,64,0,1,fp8,fp8,0,0.03121600051720937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,64,128,1,float16,float16,0,1.460960070292155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,64,0,1,float16,float16,0,1.4310347239176433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,64,128,1,float16,fp8,0,1.457914670308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,64,0,1,float16,fp8,0,1.4164212544759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,64,0,1,fp8,fp8,0,1.3119146823883057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,64,0,1,float16,float16,0,1.438096046447754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,64,128,1,float16,float16,0,1.4870239893595378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,64,128,1,fp8,fp8,0,1.9729973475138347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,64,128,1,float16,fp8,0,1.4723893801371257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,64,0,1,float16,fp8,0,1.4307626088460286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,64,0,1,fp8,fp8,0,1.345802625020345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,64,128,1,fp8,fp8,0,2.00435733795166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,64,0,1,float16,float16,0,1.4940640131632488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,64,128,1,float16,float16,0,1.5289012591044109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,64,128,1,float16,fp8,0,1.5172746976216633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,64,0,1,fp8,fp8,0,1.3916160265604656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,64,0,1,float16,fp8,0,1.488378683725993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,64,128,1,fp8,fp8,0,2.0485547383626304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,64,128,1,float16,float16,0,1.551786740620931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,64,0,1,float16,float16,0,1.5162773132324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,64,128,1,float16,fp8,0,1.5485226313273113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,64,128,1,fp8,fp8,0,2.1234985987345376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,64,128,1,float16,float16,0,0.9177866776784261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,64,0,1,float16,float16,0,0.9040319919586182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,64,128,1,float16,fp8,0,0.8835893472035726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,64,0,1,float16,fp8,0,1.534538745880127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,64,0,1,fp8,fp8,0,1.4454560279846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,64,128,1,fp8,fp8,0,1.101962645848592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,64,0,1,float16,fp8,0,0.8711360295613607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,64,0,1,fp8,fp8,0,0.7497386932373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,64,128,1,float16,float16,0,0.7470719814300537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,64,0,1,float16,float16,0,0.7150239944458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,64,128,1,float16,fp8,0,0.7432106335957845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,64,0,1,float16,fp8,0,0.7248319784800211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,64,128,1,fp8,fp8,0,1.0011413097381592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,64,0,1,fp8,fp8,0,0.6606719891230265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,64,128,1,float16,float16,0,0.7386399904886881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,64,0,1,float16,float16,0,0.7312053044637045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,64,128,1,float16,fp8,0,0.7485919793446859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,64,128,1,fp8,fp8,0,1.0238133271535237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,64,0,1,float16,fp8,0,0.7321333090464274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,64,0,1,fp8,fp8,0,0.6660266717274984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,64,128,1,float16,float16,0,0.7644373575846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,64,0,1,float16,float16,0,0.7420746485392252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,64,128,1,float16,fp8,0,0.7724533081054688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,64,128,1,fp8,fp8,0,1.0388533274332683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,64,0,1,float16,fp8,0,0.7454346815745035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,64,0,1,fp8,fp8,0,0.703162670135498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,64,128,1,float16,float16,0,0.764469305674235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,64,0,1,float16,float16,0,0.7500800291697184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,64,128,1,float16,fp8,0,0.7639040152231852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,64,128,1,float16,float16,0,0.4726346731185913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,64,0,1,float16,float16,0,0.46581868330637616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,64,0,1,float16,fp8,0,0.7559093634287516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,64,0,1,fp8,fp8,0,0.7194133599599203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,64,128,1,float16,fp8,0,0.45687464872996014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,64,128,1,fp8,fp8,0,1.054853359858195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,64,128,1,fp8,fp8,0,0.5489333470662435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,64,0,1,float16,float16,0,0.37174399693806964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,64,0,1,float16,fp8,0,0.45028265317281085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,64,0,1,fp8,fp8,0,0.3782026767730713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,64,128,1,float16,float16,0,0.38253867626190186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,64,128,1,float16,fp8,0,0.3763200044631958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,64,128,1,fp8,fp8,0,0.5149439970652262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,64,0,1,float16,fp8,0,0.36527466773986816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,64,0,1,fp8,fp8,0,0.3401706616083781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,64,128,1,float16,float16,0,0.3856000105539958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,64,0,1,float16,float16,0,0.37542398770650226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,64,128,1,float16,fp8,0,0.3853120009104411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,64,0,1,float16,fp8,0,0.37040531635284424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,64,128,1,fp8,fp8,0,0.5189333359400431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,64,0,1,fp8,fp8,0,0.34730132420857746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,64,128,1,float16,float16,0,0.3893866539001465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,64,0,1,float16,float16,0,0.3823946714401245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,64,128,1,float16,fp8,0,0.3958880106608073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,64,0,1,float16,float16,0,0.386405348777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,64,128,1,fp8,fp8,0,0.5224639972050985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,64,0,1,float16,fp8,0,0.38674668471018475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,64,0,1,fp8,fp8,0,0.35125335057576496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,64,128,1,float16,float16,0,0.25041600068410236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,64,128,1,float16,float16,0,0.3940746784210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,64,128,1,float16,fp8,0,0.38945066928863525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,64,0,1,float16,fp8,0,0.3877280155817668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,64,128,1,fp8,fp8,0,0.5360426505406698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,64,0,1,fp8,fp8,0,0.3680373430252075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,64,0,1,float16,float16,0,0.24728532632191977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,64,128,1,float16,fp8,0,0.2428213357925415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,64,128,1,fp8,fp8,0,0.2834453384081523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,64,0,1,fp8,fp8,0,0.20287466049194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,64,0,1,float16,fp8,0,0.24116800228754678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,64,128,1,fp8,fp8,0,0.2648800015449524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,64,128,1,float16,float16,0,0.19947733481725058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,64,128,1,float16,float16,0,0.20295999447504678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,64,0,1,float16,float16,0,0.1930239995320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,64,128,1,float16,fp8,0,0.19996267557144165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,64,0,1,float16,fp8,0,0.19341866175333658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,64,0,1,fp8,fp8,0,0.18099733193715414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,64,128,1,float16,fp8,0,0.20121600230534872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,64,0,1,float16,float16,0,0.19633599122365317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,64,0,1,float16,fp8,0,0.19790399074554443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,64,128,1,fp8,fp8,0,0.2676746646563212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,64,0,1,fp8,fp8,0,0.1820853352546692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,64,128,1,float16,float16,0,0.20658665895462036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,64,0,1,float16,float16,0,0.2013173302014669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,64,128,1,float16,fp8,0,0.2060746749242147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,64,128,1,fp8,fp8,0,0.26895467440287274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,64,0,1,float16,fp8,0,0.20244266589482626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,64,0,1,fp8,fp8,0,0.18838934103647867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,64,0,1,float16,fp8,0,0.20360000928243002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,64,128,1,float16,float16,0,0.2089759906133016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,64,0,1,float16,float16,0,0.20402133464813232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,64,128,1,float16,fp8,0,0.20825600624084473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,64,128,1,fp8,fp8,0,0.2756693363189697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,64,0,1,fp8,fp8,0,0.1930720011393229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,64,128,1,float16,float16,0,0.1388159990310669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,64,0,1,float16,float16,0,0.1366933286190033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,64,128,1,float16,fp8,0,0.13644267121950784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,64,128,1,fp8,fp8,0,0.1536799967288971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,64,0,1,float16,fp8,0,0.13493333260218301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,64,0,1,fp8,fp8,0,0.11341866850852966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,64,128,1,float16,float16,0,0.10605866710344951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,64,0,1,float16,float16,0,0.10285866260528564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,64,128,1,float16,fp8,0,0.10609599947929382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,64,128,1,fp8,fp8,0,0.14205333590507507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,64,0,1,float16,fp8,0,0.10355732838312785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,64,0,1,fp8,fp8,0,0.10010133186976115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,64,128,1,float16,float16,0,0.10591466228167216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,64,0,1,float16,float16,0,0.10388799508412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,64,128,1,float16,fp8,0,0.10628267129262288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,64,128,1,fp8,fp8,0,0.13929599523544312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,64,0,1,float16,fp8,0,0.10355732838312785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,64,0,1,fp8,fp8,0,0.09969066580136617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,64,128,1,float16,float16,0,0.10949333508809407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,64,0,1,float16,float16,0,0.10632000366846721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,64,128,1,float16,fp8,0,0.10973866780598958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,64,128,1,fp8,fp8,0,0.14461333552996317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,64,0,1,float16,fp8,0,0.10629333058993022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,64,0,1,fp8,fp8,0,0.10254933436711629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,64,128,1,float16,float16,0,0.1120799978574117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,64,0,1,float16,float16,0,0.11017599701881409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,64,128,1,float16,fp8,0,0.11142399907112122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,64,0,1,float16,float16,0,0.0813920001188914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,64,128,1,fp8,fp8,0,0.14620799819628397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,64,0,1,float16,fp8,0,0.11000000437100728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,64,0,1,fp8,fp8,0,0.1043893297513326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,64,128,1,float16,float16,0,0.08283733328183492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,64,128,1,float16,fp8,0,0.0812853326400121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,64,128,1,fp8,fp8,0,0.08691199620564778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,64,0,1,float16,fp8,0,0.07923200229803722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,64,0,1,fp8,fp8,0,0.06621866424878438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,64,128,1,float16,float16,0,0.06189866860707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,64,0,1,float16,float16,0,0.062421331803003945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,64,128,1,float16,fp8,0,0.06274133423964183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,64,128,1,fp8,fp8,0,0.07720000048478444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,64,0,1,float16,fp8,0,0.06204266846179962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,64,0,1,fp8,fp8,0,0.0562720000743866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,64,128,1,float16,float16,0,0.06181866427262624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,64,0,1,float16,float16,0,0.06282666822274525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,64,128,1,float16,fp8,0,0.06423466900984447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,64,0,1,float16,float16,0,0.06384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,64,128,1,fp8,fp8,0,0.07791466514269511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,64,0,1,float16,fp8,0,0.06328533093134563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,64,0,1,fp8,fp8,0,0.05905066430568695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,64,128,1,float16,float16,0,0.06374933322270711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,64,128,1,float16,fp8,0,0.06467199822266896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,64,128,1,fp8,fp8,0,0.08011200030644734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,64,0,1,float16,fp8,0,0.0642133355140686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,64,0,1,fp8,fp8,0,0.06062399844328562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,64,128,1,float16,float16,0,0.0660159985224406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,64,0,1,float16,float16,0,0.06640533109505971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,64,128,1,fp8,fp8,0,0.08249066770076752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,64,128,1,float16,fp8,0,0.06657599906126659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,64,0,1,float16,fp8,0,0.06428266565004985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,64,0,1,fp8,fp8,0,0.06039999922116598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,64,128,1,float16,float16,0,0.04298666616280874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,64,0,1,float16,float16,0,0.041589332123597465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,64,128,1,float16,fp8,0,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,64,128,1,fp8,fp8,0,0.049866666396458946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,64,0,1,float16,fp8,0,0.041237334410349526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,64,0,1,fp8,fp8,0,0.03937600056330363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,64,128,1,float16,float16,0,0.037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,64,0,1,float16,float16,0,0.03713600089152654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,64,128,1,float16,fp8,0,0.03807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,64,128,1,fp8,fp8,0,0.04541866481304169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,64,128,1,fp8,fp8,0,0.04580800235271454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,64,0,1,float16,fp8,0,0.037445334096749626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,64,0,1,fp8,fp8,0,0.03580799947182337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,64,0,1,float16,fp8,0,0.03707200040419897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,64,0,1,fp8,fp8,0,0.03589866558710734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,64,128,1,float16,float16,0,0.03941333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,64,0,1,float16,float16,0,0.03700266778469086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,64,128,1,float16,fp8,0,0.038058665891488395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,64,128,1,float16,float16,0,0.03867200016975403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,64,0,1,float16,float16,0,0.037733333806196846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,64,128,1,float16,fp8,0,0.03942399968703588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,64,128,1,fp8,fp8,0,0.047397335370381675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,64,0,1,float16,fp8,0,0.03814399987459183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,64,0,1,fp8,fp8,0,0.03695466617743174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,64,128,1,float16,float16,0,0.039221333960692085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,64,0,1,float16,float16,0,0.03825599948565165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,64,128,1,float16,fp8,0,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,64,128,1,fp8,fp8,0,0.046767999728520714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,64,0,1,float16,fp8,0,0.03822933385769526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,64,0,1,fp8,fp8,0,0.03699733316898346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,64,128,1,float16,float16,0,0.031157332162062328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,64,0,1,float16,float16,0,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,64,128,1,float16,fp8,0,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,64,128,1,fp8,fp8,0,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,64,0,1,float16,fp8,0,0.030965333183606465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,64,0,1,fp8,fp8,0,0.02905600021282832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,64,128,1,float16,float16,0,0.028618666032950085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,64,0,1,float16,float16,0,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,64,128,1,float16,fp8,0,0.028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,64,128,1,fp8,fp8,0,0.034474665919939675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,64,0,1,float16,fp8,0,0.02829333394765854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,64,0,1,fp8,fp8,0,0.02734400083621343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,64,128,1,float16,float16,0,0.02828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,64,0,1,float16,float16,0,0.027776000400384266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,64,128,1,float16,fp8,0,0.028325334191322327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,64,128,1,fp8,fp8,0,0.03460799902677536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,64,0,1,float16,fp8,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,64,0,1,fp8,fp8,0,0.028090665737787884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,64,128,1,float16,float16,0,0.02932800104220708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,64,0,1,float16,float16,0,0.02850666642189026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,64,128,1,float16,fp8,0,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,64,128,1,fp8,fp8,0,0.034602666894594826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,64,0,1,float16,fp8,0,0.02863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,64,0,1,fp8,fp8,0,0.02805333336194356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,64,128,1,float16,float16,0,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,64,0,1,float16,float16,0,0.028954667349656422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,64,128,1,float16,fp8,0,0.029461334149042766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,64,128,1,fp8,fp8,0,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,64,128,1,fp8,fp8,0,0.024154665569464367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,64,0,1,float16,fp8,0,0.02847466617822647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,64,0,1,fp8,fp8,0,0.028714666763941448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,64,128,1,float16,float16,0,0.019978666057189304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,64,0,1,float16,float16,0,0.019871999820073444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,64,128,1,float16,fp8,0,0.01860800012946129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,64,128,1,float16,fp8,0,0.02094399929046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,64,0,1,float16,fp8,0,0.02025066688656807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,64,0,1,fp8,fp8,0,0.020666666328907013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,64,128,1,float16,float16,0,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,64,0,1,float16,float16,0,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,64,128,1,fp8,fp8,0,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,64,0,1,float16,fp8,0,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,64,0,1,fp8,fp8,0,0.019882666567961376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,64,128,1,float16,float16,0,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,64,0,1,float16,float16,0,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,64,128,1,float16,fp8,0,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,64,128,1,fp8,fp8,0,0.023557332654794056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,64,0,1,float16,fp8,0,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,64,0,1,fp8,fp8,0,0.020154666155576706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,64,128,1,float16,float16,0,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,64,0,1,float16,float16,0,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,64,128,1,float16,fp8,0,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,64,128,1,fp8,fp8,0,0.023733332753181458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,64,0,1,float16,fp8,0,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,64,0,1,fp8,fp8,0,0.019567999988794327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,64,0,1,fp8,fp8,0,0.020010666300853092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,64,128,1,float16,float16,0,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,64,0,1,float16,float16,0,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,64,128,1,float16,fp8,0,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,64,0,1,float16,float16,0,0.541045347849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,64,128,1,fp8,fp8,0,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,64,0,1,float16,fp8,0,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,64,128,1,float16,float16,0,0.5415466626485189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,64,128,1,float16,fp8,0,0.5383893251419067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,64,128,1,fp8,fp8,0,0.6515253384908041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,64,0,1,float16,fp8,0,0.5409066677093506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,64,0,1,fp8,fp8,0,0.6516426801681519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,64,128,1,float16,float16,0,0.5465013186136881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,64,128,1,float16,fp8,0,0.5401440064112345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,64,0,1,float16,float16,0,0.5503093401590983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,64,128,1,fp8,fp8,0,0.6490346590677897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,64,0,1,float16,fp8,0,0.5427253246307373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,64,0,1,fp8,fp8,0,0.6497919956843058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,64,128,1,float16,float16,0,0.5784639914830526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,64,0,1,float16,float16,0,0.5769973198572794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,64,128,1,float16,fp8,0,0.566373348236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,64,0,1,float16,fp8,0,0.5702826579411825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,64,128,1,fp8,fp8,0,0.7013653119405111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,64,128,1,float16,float16,0,0.5816053152084351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,64,0,1,fp8,fp8,0,0.6996959845225016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,64,0,1,float16,float16,0,0.5840799808502197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,64,128,1,float16,fp8,0,0.57532266775767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,64,128,1,float16,float16,0,0.3863519827524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,64,0,1,float16,fp8,0,0.5693866809209188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,64,0,1,fp8,fp8,0,0.705888032913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,64,128,1,fp8,fp8,0,0.7092800140380859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,64,0,1,float16,float16,0,0.3869493405024211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,64,128,1,float16,fp8,0,0.37299732367197674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,64,128,1,fp8,fp8,0,0.3839626709620158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,64,0,1,float16,fp8,0,0.3733760118484497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,64,128,1,float16,fp8,0,0.273418664932251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,64,0,1,fp8,fp8,0,0.3832480112711589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,64,128,1,float16,float16,0,0.2762773235638936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,64,0,1,float16,float16,0,0.27695999542872113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,64,128,1,fp8,fp8,0,0.33455467224121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,64,0,1,float16,fp8,0,0.27399466435114544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,64,0,1,fp8,fp8,0,0.3331786592801412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,64,128,1,float16,float16,0,0.2810506621996562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,64,0,1,float16,float16,0,0.27692800760269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,64,128,1,float16,fp8,0,0.27530133724212646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,64,128,1,fp8,fp8,0,0.3283146619796753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,64,0,1,float16,fp8,0,0.276911993821462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,64,0,1,fp8,fp8,0,0.3285973270734151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,64,128,1,float16,float16,0,0.2930506666501363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,64,0,1,float16,float16,0,0.2909440000851949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,64,0,1,float16,fp8,0,0.28949334224065143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,64,128,1,float16,fp8,0,0.29099732637405396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,64,128,1,fp8,fp8,0,0.35762667655944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,64,0,1,fp8,fp8,0,0.3531786600748698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,64,128,1,float16,float16,0,0.29318400224049884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,64,0,1,float16,float16,0,0.29267199834187824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,64,128,1,float16,fp8,0,0.2913173238436381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,64,128,1,fp8,fp8,0,0.3587840000788371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,64,128,1,float16,float16,0,0.2047040065129598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,64,0,1,float16,fp8,0,0.2911093235015869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,64,0,1,fp8,fp8,0,0.3617226680119832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,64,0,1,float16,float16,0,0.20408000548680624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,64,128,1,float16,fp8,0,0.19849065939585367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,64,128,1,fp8,fp8,0,0.20137600104014078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,64,128,1,float16,fp8,0,0.14607999722162882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,64,128,1,fp8,fp8,0,0.179584006468455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,64,0,1,float16,fp8,0,0.19798400004704794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,64,0,1,fp8,fp8,0,0.19993066787719727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,64,128,1,float16,float16,0,0.14630400141080221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,64,0,1,float16,float16,0,0.14731199542681375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,64,0,1,float16,fp8,0,0.14595199624697366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,64,0,1,fp8,fp8,0,0.17776532967885336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,64,128,1,float16,float16,0,0.1486133337020874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,64,0,1,float16,float16,0,0.1481119990348816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,64,128,1,float16,fp8,0,0.14684266845385233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,64,128,1,fp8,fp8,0,0.17727466424306235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,64,0,1,float16,fp8,0,0.14822933077812195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,64,0,1,fp8,fp8,0,0.1781760056813558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,64,128,1,float16,float16,0,0.15366933743158975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,64,0,1,float16,float16,0,0.15382933616638184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,64,128,1,float16,fp8,0,0.15261866648991904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,64,128,1,fp8,fp8,0,0.18597332636515299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,64,0,1,float16,fp8,0,0.1534346640110016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,64,0,1,fp8,fp8,0,0.18734399477640787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,64,128,1,float16,float16,0,0.15532267093658447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,64,0,1,float16,float16,0,0.15585066874821982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,64,128,1,float16,fp8,0,0.15408533811569214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,64,128,1,fp8,fp8,0,0.18807466824849448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,64,0,1,float16,fp8,0,0.1539253294467926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,64,128,1,float16,fp8,0,0.10919466614723206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,64,128,1,fp8,fp8,0,0.11209600170453389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,64,0,1,fp8,fp8,0,0.18844266732533774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,64,0,1,float16,fp8,0,0.10957333445549011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,64,128,1,float16,float16,0,0.11314666271209717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,64,0,1,float16,float16,0,0.11296533544858296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,64,0,1,fp8,fp8,0,0.11166399717330933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,64,128,1,float16,float16,0,0.08045333127180736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,64,0,1,float16,float16,0,0.08032533526420593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,64,128,1,float16,fp8,0,0.08083733419577281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,64,128,1,fp8,fp8,0,0.09842666983604431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,64,0,1,float16,float16,0,0.08041599889596303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,64,0,1,float16,fp8,0,0.08060266574223836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,64,0,1,fp8,fp8,0,0.1006719966729482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,64,128,1,float16,float16,0,0.08051733175913493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,64,128,1,float16,fp8,0,0.081535999973615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,64,128,1,fp8,fp8,0,0.09834667046864827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,64,0,1,float16,fp8,0,0.08108800152937572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,64,0,1,fp8,fp8,0,0.09905067086219788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,64,0,1,float16,fp8,0,0.0832586685816447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,64,128,1,float16,float16,0,0.08422399560610454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,64,0,1,float16,float16,0,0.08449600140253703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,64,128,1,float16,fp8,0,0.08295466502507527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,64,0,1,float16,fp8,0,0.0862613320350647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,64,128,1,fp8,fp8,0,0.10188266634941101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,64,128,1,float16,float16,0,0.0860640009244283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,64,0,1,fp8,fp8,0,0.1018933355808258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,64,0,1,float16,float16,0,0.08572266499201457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,64,128,1,float16,fp8,0,0.08597333232561748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,64,128,1,fp8,fp8,0,0.10428800185521443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,64,0,1,fp8,fp8,0,0.10381333033243816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,64,128,1,float16,float16,0,0.06337599953015645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,64,0,1,float16,float16,0,0.06413333117961884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,64,128,1,float16,fp8,0,0.061610668897628784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,64,128,1,fp8,fp8,0,0.06582933167616527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,64,128,1,fp8,fp8,0,0.05665066838264465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,64,0,1,float16,fp8,0,0.06121066709359487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,64,0,1,fp8,fp8,0,0.0653546651204427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,64,128,1,float16,float16,0,0.04602666695912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,64,0,1,float16,float16,0,0.046816001335779824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,64,128,1,float16,fp8,0,0.0476693312327067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,64,0,1,float16,float16,0,0.04601066807905833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,64,128,1,float16,fp8,0,0.046469335754712425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,64,0,1,float16,fp8,0,0.045882667104403176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,64,0,1,fp8,fp8,0,0.05681600173314413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,64,128,1,float16,float16,0,0.047354668378829956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,64,128,1,fp8,fp8,0,0.05834666887919108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,64,0,1,float16,fp8,0,0.04741866886615753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,64,0,1,fp8,fp8,0,0.05750933289527893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,64,128,1,float16,float16,0,0.04921066761016846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,64,128,1,float16,float16,0,0.04964800179004669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,64,0,1,float16,float16,0,0.04891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,64,128,1,float16,fp8,0,0.04866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,64,128,1,fp8,fp8,0,0.05942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,64,0,1,float16,fp8,0,0.0488373339176178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,64,0,1,fp8,fp8,0,0.06052800019582113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,64,0,1,fp8,fp8,0,0.05959466596444448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,64,0,1,float16,float16,0,0.049365331729253135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,64,128,1,float16,fp8,0,0.050527999798456825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,64,128,1,fp8,fp8,0,0.061381335059801735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,64,0,1,float16,fp8,0,0.049440001447995506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,64,128,1,float16,float16,0,0.03226666649182638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,64,0,1,float16,float16,0,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,64,128,1,float16,fp8,0,0.03146666785081228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,64,128,1,fp8,fp8,0,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,64,0,1,float16,fp8,0,0.030794667700926464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,64,0,1,fp8,fp8,0,0.03828266759713491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,64,128,1,float16,float16,0,0.027957332630952198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,64,0,1,float16,float16,0,0.027610667049884796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,64,128,1,float16,fp8,0,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,64,128,1,fp8,fp8,0,0.03521066655715307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,64,0,1,float16,fp8,0,0.027989332874615986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,64,0,1,fp8,fp8,0,0.03509866694609324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,64,128,1,float16,float16,0,0.02826133370399475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,64,0,1,float16,float16,0,0.027845333019892376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,64,128,1,float16,fp8,0,0.028650666276613872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,64,128,1,fp8,fp8,0,0.03555200000603994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,64,0,1,float16,fp8,0,0.02938133229811986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,64,0,1,fp8,fp8,0,0.03621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,64,128,1,float16,float16,0,0.028592000404993694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,64,0,1,float16,float16,0,0.028586665789286297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,64,128,1,float16,fp8,0,0.0296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,64,128,1,fp8,fp8,0,0.03717333326737086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,64,128,1,float16,fp8,0,0.02940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,64,0,1,float16,fp8,0,0.029189333319664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,64,0,1,fp8,fp8,0,0.03707200040419897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,64,128,1,float16,float16,0,0.029680001238981884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,64,128,1,float16,float16,0,0.024314666787783306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,64,0,1,float16,float16,0,0.028954667349656422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,64,128,1,fp8,fp8,0,0.03654933224121729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,64,0,1,float16,fp8,0,0.02976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,64,0,1,fp8,fp8,0,0.03623999903599421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,64,0,1,fp8,fp8,0,0.02903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,64,0,1,float16,float16,0,0.024330665667851765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,64,128,1,float16,fp8,0,0.02367999901374181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,64,128,1,fp8,fp8,0,0.02868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,64,0,1,float16,fp8,0,0.023936000963052113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,64,128,1,float16,float16,0,0.022053333620230358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,64,0,1,float16,float16,0,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,64,128,1,float16,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,64,128,1,fp8,fp8,0,0.026901334524154663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,64,0,1,float16,fp8,0,0.02179733415444692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,64,0,1,fp8,fp8,0,0.02755733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,64,128,1,float16,float16,0,0.021717332303524017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,64,0,1,float16,float16,0,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,64,128,1,float16,fp8,0,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,64,128,1,fp8,fp8,0,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,64,0,1,float16,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,64,0,1,fp8,fp8,0,0.02758399893840154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,64,128,1,float16,float16,0,0.02252800017595291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,64,0,1,float16,float16,0,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,64,128,1,float16,fp8,0,0.022533332308133442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,64,128,1,fp8,fp8,0,0.02808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,64,0,1,float16,fp8,0,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,64,0,1,fp8,fp8,0,0.028197333216667175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,64,128,1,float16,float16,0,0.022341333329677582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,64,0,1,float16,float16,0,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,64,128,1,float16,fp8,0,0.022944000860055287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,64,128,1,fp8,fp8,0,0.028378665447235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,64,0,1,float16,fp8,0,0.02250133454799652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,64,0,1,fp8,fp8,0,0.028058665494124096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,64,128,1,float16,float16,0,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,64,0,1,float16,float16,0,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,64,128,1,float16,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,64,128,1,fp8,fp8,0,0.020501332978407543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,64,0,1,float16,fp8,0,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,64,0,1,fp8,fp8,0,0.02056533346573512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,64,128,1,float16,float16,0,0.015664000064134598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,64,0,1,float16,float16,0,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,64,128,1,float16,fp8,0,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,64,128,1,fp8,fp8,0,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,64,0,1,float16,fp8,0,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,64,0,1,float16,fp8,0,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,64,0,1,fp8,fp8,0,0.020015999674797058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,64,128,1,float16,float16,0,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,64,0,1,float16,float16,0,0.016048000504573185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,64,128,1,float16,fp8,0,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,64,128,1,fp8,fp8,0,0.01988799994190534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,64,0,1,fp8,fp8,0,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,64,128,1,float16,float16,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,64,0,1,float16,float16,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,64,128,1,float16,fp8,0,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,64,128,1,fp8,fp8,0,0.019653332730134327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,64,0,1,float16,fp8,0,0.01634666696190834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,64,0,1,float16,fp8,0,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,64,0,1,fp8,fp8,0,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,64,128,1,float16,float16,0,0.016250666230916977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,64,0,1,float16,float16,0,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,64,128,1,float16,fp8,0,0.016336000214020412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,64,128,1,fp8,fp8,0,0.020362666497627895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,64,0,1,fp8,fp8,0,0.019727999965349834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,64,128,1,float16,float16,0,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,64,0,1,float16,float16,0,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,64,128,1,float16,fp8,0,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,64,128,1,fp8,fp8,0,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,64,0,1,float16,fp8,0,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,64,0,1,fp8,fp8,0,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,64,128,1,float16,float16,0,0.013888000200192133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,64,0,1,float16,float16,0,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,64,128,1,float16,fp8,0,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,64,128,1,fp8,fp8,0,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,64,0,1,float16,fp8,0,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,64,0,1,fp8,fp8,0,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,64,128,1,float16,float16,0,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,64,0,1,float16,float16,0,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,64,128,1,float16,fp8,0,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,64,128,1,fp8,fp8,0,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,64,0,1,float16,fp8,0,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,64,0,1,fp8,fp8,0,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,64,128,1,float16,float16,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,64,0,1,float16,float16,0,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,64,128,1,float16,fp8,0,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,64,128,1,fp8,fp8,0,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,64,0,1,float16,fp8,0,0.014261333892742792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,64,0,1,fp8,fp8,0,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,64,128,1,float16,float16,0,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,64,0,1,float16,float16,0,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,64,128,1,float16,fp8,0,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,64,128,1,fp8,fp8,0,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,64,0,1,float16,fp8,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,64,0,1,fp8,fp8,0,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,64,128,1,float16,float16,0,0.260368009408315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,64,0,1,float16,float16,0,0.26057066520055133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,64,128,1,float16,fp8,0,0.2579626639684041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,64,0,1,float16,fp8,0,0.25917333364486694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,64,128,1,fp8,fp8,0,0.4511200189590454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,64,128,1,float16,float16,0,0.2627999981244405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,64,0,1,float16,float16,0,0.2625333269437154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,64,128,1,float16,fp8,0,0.2568639914194743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,64,0,1,fp8,fp8,0,0.44968533515930176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,64,128,1,fp8,fp8,0,0.4488159815470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,64,0,1,float16,fp8,0,0.25811733802159625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,64,128,1,float16,float16,0,0.2725813388824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,64,0,1,fp8,fp8,0,0.4514506657918294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,64,0,1,float16,float16,0,0.2711733380953471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,64,128,1,float16,fp8,0,0.27105067173639935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,64,0,1,float16,fp8,0,0.27026132742563885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,64,128,1,float16,float16,0,0.2737813393274943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,64,0,1,fp8,fp8,0,0.4768426815668742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,64,128,1,fp8,fp8,0,0.475600004196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,64,0,1,float16,float16,0,0.27476799488067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,64,128,1,float16,fp8,0,0.273525337378184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,64,128,1,fp8,fp8,0,0.47860801219940186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,64,0,1,float16,fp8,0,0.27265600363413495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,64,128,1,float16,float16,0,0.20067733526229858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,64,0,1,fp8,fp8,0,0.4758880138397217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,64,0,1,fp8,fp8,0,0.2606400052706401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,64,0,1,float16,float16,0,0.20243734121322632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,64,128,1,float16,fp8,0,0.1939093271891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,64,128,1,fp8,fp8,0,0.25998934110005695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,64,0,1,float16,fp8,0,0.19350399573644003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,64,128,1,float16,float16,0,0.1378666659196218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,64,0,1,float16,float16,0,0.1381066640218099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,64,128,1,float16,fp8,0,0.13802666465441385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,64,0,1,float16,fp8,0,0.13850667079289755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,64,128,1,fp8,fp8,0,0.23733333746592203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,64,0,1,fp8,fp8,0,0.23827733596165976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,64,128,1,float16,float16,0,0.1384160021940867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,64,0,1,float16,float16,0,0.1393226683139801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,64,128,1,float16,fp8,0,0.13822933038075766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,64,128,1,float16,fp8,0,0.14341866970062256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,64,128,1,fp8,fp8,0,0.23509333531061807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,64,0,1,float16,fp8,0,0.13865066568056741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,64,128,1,fp8,fp8,0,0.24578134218851724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,64,0,1,fp8,fp8,0,0.2360853354136149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,64,128,1,float16,float16,0,0.14461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,64,0,1,float16,float16,0,0.1434399982293447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,64,0,1,float16,fp8,0,0.1436853309472402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,64,0,1,fp8,fp8,0,0.24384532372156778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,64,128,1,float16,float16,0,0.14456533392270407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,64,0,1,float16,float16,0,0.14524267117182413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,64,128,1,float16,fp8,0,0.10598400235176086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,64,128,1,float16,fp8,0,0.14662933349609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,64,128,1,fp8,fp8,0,0.24963200092315674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,64,0,1,float16,fp8,0,0.14411733547846475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,64,128,1,float16,float16,0,0.10893332958221436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,64,0,1,fp8,fp8,0,0.2492213249206543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,64,0,1,float16,float16,0,0.07482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,64,0,1,float16,float16,0,0.10852799812952678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,64,128,1,fp8,fp8,0,0.14073066910107931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,64,0,1,float16,fp8,0,0.10548800230026245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,64,0,1,fp8,fp8,0,0.14197867115338644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,64,128,1,float16,float16,0,0.07520000139872234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,64,128,1,float16,fp8,0,0.07505600154399872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,64,128,1,fp8,fp8,0,0.12771200140317282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,64,0,1,float16,fp8,0,0.07507200042406718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,64,0,1,fp8,fp8,0,0.12723199526468912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,64,128,1,float16,float16,0,0.075914666056633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,64,0,1,float16,float16,0,0.07579199969768524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,64,128,1,float16,fp8,0,0.07638399799664815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,64,0,1,float16,float16,0,0.07992533346017201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,64,128,1,fp8,fp8,0,0.12771733601888022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,64,0,1,float16,fp8,0,0.07521600027879079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,64,0,1,fp8,fp8,0,0.12761599818865457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,64,128,1,float16,float16,0,0.07966400186220805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,64,128,1,float16,fp8,0,0.07935999830563863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,64,128,1,fp8,fp8,0,0.13182933131853738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,64,0,1,float16,fp8,0,0.07969599962234497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,64,0,1,fp8,fp8,0,0.1320853332678477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,64,128,1,float16,float16,0,0.08096000055472057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,64,0,1,fp8,fp8,0,0.13311466574668884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,64,0,1,float16,float16,0,0.0803466687599818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,64,0,1,float16,float16,0,0.0603413333495458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,64,128,1,float16,fp8,0,0.08037333190441132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,64,128,1,fp8,fp8,0,0.13276799519856772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,64,0,1,float16,fp8,0,0.0801386684179306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,64,128,1,float16,float16,0,0.06052266558011373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,64,128,1,float16,fp8,0,0.05759466687838236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,64,128,1,fp8,fp8,0,0.08091733356316884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,64,0,1,float16,fp8,0,0.05766933163007101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,64,128,1,fp8,fp8,0,0.07098133365313213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,64,0,1,fp8,fp8,0,0.07982400059700012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,64,128,1,float16,float16,0,0.043141335248947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,64,0,1,float16,float16,0,0.043391997615496315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,64,128,1,float16,fp8,0,0.04297066728274027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,64,0,1,float16,fp8,0,0.042965332667032875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,64,0,1,fp8,fp8,0,0.07208533088366191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,64,128,1,float16,float16,0,0.04347200194994608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,64,0,1,float16,float16,0,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,64,128,1,float16,fp8,0,0.04348800083001455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,64,128,1,float16,fp8,0,0.04530666768550873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,64,128,1,fp8,fp8,0,0.07120533287525177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,64,0,1,float16,fp8,0,0.043194666504859924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,64,0,1,fp8,fp8,0,0.07229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,64,0,1,fp8,fp8,0,0.07407466570536296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,64,128,1,float16,float16,0,0.046165332198143005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,64,0,1,float16,float16,0,0.04507199923197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,64,128,1,fp8,fp8,0,0.07483733197053273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,64,0,1,float16,fp8,0,0.045237332582473755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,64,128,1,float16,float16,0,0.04568000137805939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,64,0,1,float16,float16,0,0.04538666705290476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,64,128,1,float16,fp8,0,0.045797333121299744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,64,128,1,fp8,fp8,0,0.07554666697978973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,64,0,1,float16,fp8,0,0.04613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,64,0,1,fp8,fp8,0,0.07564266522725423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,64,128,1,float16,float16,0,0.02942399928967158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,64,0,1,fp8,fp8,0,0.0461760014295578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,64,0,1,float16,float16,0,0.02887466549873352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,64,128,1,float16,fp8,0,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,64,128,1,fp8,fp8,0,0.04577599962552389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,64,0,1,float16,fp8,0,0.029018667836983997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,64,128,1,float16,float16,0,0.026389333109060924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,64,0,1,float16,float16,0,0.02632533262173335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,64,128,1,float16,fp8,0,0.02661866694688797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,64,128,1,fp8,fp8,0,0.043136000633239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,64,0,1,float16,fp8,0,0.02672533442576726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,64,0,1,fp8,fp8,0,0.042725334564844765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,64,128,1,float16,float16,0,0.02664000044266383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,64,0,1,float16,float16,0,0.026693334182103474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,64,128,1,float16,fp8,0,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,64,128,1,fp8,fp8,0,0.04398400088151296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,64,0,1,float16,fp8,0,0.026757332185904186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,64,0,1,float16,fp8,0,0.028255999088287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,64,0,1,fp8,fp8,0,0.043418665726979576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,64,128,1,float16,float16,0,0.027488000690937042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,64,128,1,float16,float16,0,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,64,0,1,float16,float16,0,0.027600000301996868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,64,128,1,float16,fp8,0,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,64,128,1,fp8,fp8,0,0.04438399771849314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,64,0,1,fp8,fp8,0,0.04465599854787191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,64,0,1,float16,float16,0,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,64,128,1,float16,fp8,0,0.02787200113137563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,64,128,1,fp8,fp8,0,0.04456533491611481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,64,0,1,float16,fp8,0,0.02775999903678894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,64,0,1,fp8,fp8,0,0.04483200112978617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,64,128,1,float16,float16,0,0.021840001145998638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,64,0,1,float16,float16,0,0.021669333179791767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,64,128,1,float16,fp8,0,0.02199466774861018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,64,128,1,fp8,fp8,0,0.0310506671667099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,64,0,1,float16,fp8,0,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,64,0,1,fp8,fp8,0,0.031119999786218006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,64,128,1,float16,float16,0,0.01982933282852173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,64,0,1,float16,float16,0,0.02013333390156428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,64,128,1,float16,fp8,0,0.02046400060256322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,64,128,1,fp8,fp8,0,0.029157333076000214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,64,0,1,float16,fp8,0,0.020181333025296528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,64,0,1,fp8,fp8,0,0.029653333127498627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,64,128,1,float16,float16,0,0.019962667177120846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,64,0,1,float16,float16,0,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,64,128,1,float16,fp8,0,0.02014933278163274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,64,128,1,fp8,fp8,0,0.02926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,64,0,1,float16,fp8,0,0.02067199970285098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,64,0,1,fp8,fp8,0,0.029578665892283123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,64,128,1,float16,float16,0,0.02085866779088974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,64,0,1,float16,float16,0,0.020538666596015293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,64,128,1,float16,fp8,0,0.020901332298914593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,64,128,1,fp8,fp8,0,0.030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,64,0,1,float16,fp8,0,0.020554666717847187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,64,0,1,float16,fp8,0,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,64,0,1,fp8,fp8,0,0.029882666965325672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,64,128,1,float16,float16,0,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,64,128,1,float16,float16,0,0.020917333662509918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,64,0,1,float16,float16,0,0.02085866779088974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,64,0,1,float16,fp8,0,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,64,128,1,float16,fp8,0,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,64,128,1,fp8,fp8,0,0.029904000461101532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,64,0,1,fp8,fp8,0,0.03052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,64,0,1,float16,float16,0,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,64,128,1,float16,fp8,0,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,64,128,1,fp8,fp8,0,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,64,0,1,fp8,fp8,0,0.02399466683467229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,64,128,1,float16,float16,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,64,0,1,float16,float16,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,64,128,1,float16,fp8,0,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,64,128,1,fp8,fp8,0,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,64,0,1,float16,fp8,0,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,64,0,1,fp8,fp8,0,0.022202665607134502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,64,128,1,float16,float16,0,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,64,0,1,float16,float16,0,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,64,128,1,float16,fp8,0,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,64,128,1,fp8,fp8,0,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,64,0,1,float16,fp8,0,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,64,0,1,fp8,fp8,0,0.023221333821614582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,64,128,1,float16,float16,0,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,64,0,1,float16,float16,0,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,64,128,1,float16,fp8,0,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,64,128,1,fp8,fp8,0,0.022704000274340313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,64,0,1,float16,fp8,0,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,64,0,1,fp8,fp8,0,0.02275199939807256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,64,128,1,float16,float16,0,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,64,0,1,float16,float16,0,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,64,128,1,float16,fp8,0,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,64,128,1,fp8,fp8,0,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,64,0,1,float16,fp8,0,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,64,0,1,fp8,fp8,0,0.02293866624434789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,64,128,1,float16,float16,0,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,64,0,1,float16,float16,0,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,64,128,1,float16,fp8,0,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,64,128,1,fp8,fp8,0,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,64,0,1,float16,fp8,0,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,64,0,1,fp8,fp8,0,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,64,128,1,float16,float16,0,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,64,0,1,float16,float16,0,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,64,128,1,float16,fp8,0,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,64,128,1,fp8,fp8,0,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,64,0,1,float16,fp8,0,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,64,0,1,fp8,fp8,0,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,64,128,1,float16,float16,0,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,64,128,1,float16,float16,0,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,64,0,1,float16,float16,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,64,0,1,float16,float16,0,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,64,128,1,float16,fp8,0,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,64,128,1,fp8,fp8,0,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,64,0,1,fp8,fp8,0,0.018650667121013004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,64,0,1,float16,fp8,0,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,64,0,1,fp8,fp8,0,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,64,128,1,float16,fp8,0,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,64,128,1,fp8,fp8,0,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,64,0,1,float16,fp8,0,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,64,128,1,float16,float16,0,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,64,0,1,float16,float16,0,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,64,128,1,float16,fp8,0,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,64,128,1,fp8,fp8,0,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,64,0,1,float16,fp8,0,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,64,0,1,fp8,fp8,0,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,64,128,1,float16,float16,0,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,64,0,1,float16,float16,0,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,64,0,1,float16,float16,0,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,64,128,1,float16,fp8,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,64,128,1,fp8,fp8,0,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,64,0,1,float16,fp8,0,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,64,0,1,fp8,fp8,0,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,64,128,1,float16,float16,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,64,128,1,float16,fp8,0,0.012159999459981918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,64,128,1,fp8,fp8,0,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,64,0,1,float16,fp8,0,0.012506666282812754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,64,0,1,fp8,fp8,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,64,128,1,float16,float16,0,0.012213333199421564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,64,0,1,float16,float16,0,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,64,128,1,float16,fp8,0,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,64,128,1,fp8,fp8,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,64,0,1,float16,fp8,0,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,64,0,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,64,128,1,float16,float16,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,64,0,1,float16,float16,0,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,64,128,1,float16,fp8,0,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,64,128,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,64,0,1,float16,fp8,0,0.012170666207869848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,64,0,1,fp8,fp8,0,0.017770666629076004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,64,128,1,float16,float16,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,64,0,1,float16,float16,0,0.011930666863918304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,64,128,1,float16,fp8,0,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,64,128,1,fp8,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,64,0,1,float16,fp8,0,0.012495999534924826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,64,0,1,fp8,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,64,128,1,float16,float16,0,0.15366400281588236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,64,0,1,float16,float16,0,0.15310399731000265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,64,128,1,float16,fp8,0,0.15344533324241638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,64,128,1,fp8,fp8,0,0.3737706740697225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,64,0,1,float16,fp8,0,0.15382400155067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,64,0,1,fp8,fp8,0,0.377621332804362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,64,128,1,float16,float16,0,0.15268799662590027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,64,0,1,float16,float16,0,0.15350932876269022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,64,128,1,float16,fp8,0,0.15493866801261902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,64,0,1,float16,fp8,0,0.15254933635393778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,64,128,1,fp8,fp8,0,0.37457601229349774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,64,0,1,fp8,fp8,0,0.3735733429590861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,64,128,1,float16,float16,0,0.16009599963823953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,64,0,1,float16,float16,0,0.16051733493804932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,64,128,1,float16,fp8,0,0.15794666608174643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,64,128,1,fp8,fp8,0,0.38440001010894775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,64,0,1,float16,fp8,0,0.15810666481653848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,64,128,1,float16,fp8,0,0.16274133324623108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,64,0,1,fp8,fp8,0,0.38334933916727704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,64,128,1,float16,float16,0,0.16234133640925089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,64,0,1,float16,float16,0,0.1620693306128184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,64,0,1,float16,fp8,0,0.1614400049050649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,64,128,1,fp8,fp8,0,0.3868480126063029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,64,128,1,float16,float16,0,0.10938666264216106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,64,0,1,float16,float16,0,0.11039466659228007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,64,128,1,float16,fp8,0,0.10631466905275981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,64,0,1,fp8,fp8,0,0.38818132877349854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,64,128,1,fp8,fp8,0,0.21163199345270792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,64,0,1,float16,fp8,0,0.10624000430107117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,64,128,1,float16,float16,0,0.08175999919573466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,64,0,1,fp8,fp8,0,0.21126399437586466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,64,0,1,float16,float16,0,0.08245866497357686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,64,128,1,float16,fp8,0,0.08133866886297862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,64,128,1,fp8,fp8,0,0.1974239945411682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,64,0,1,float16,fp8,0,0.08142399787902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,64,0,1,fp8,fp8,0,0.1971199909845988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,64,128,1,float16,float16,0,0.0827466646830241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,64,0,1,float16,float16,0,0.0811466674009959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,64,128,1,float16,fp8,0,0.08134399851163228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,64,128,1,fp8,fp8,0,0.19684267044067383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,64,0,1,float16,fp8,0,0.08203200002511342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,64,0,1,fp8,fp8,0,0.1965013345082601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,64,128,1,float16,float16,0,0.08649599552154541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,64,0,1,float16,float16,0,0.08724799752235413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,64,128,1,float16,fp8,0,0.0848640004793803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,64,128,1,fp8,fp8,0,0.20283200343449911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,64,0,1,float16,fp8,0,0.08521067102750142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,64,128,1,float16,float16,0,0.0886400043964386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,64,0,1,fp8,fp8,0,0.20053333044052124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,64,0,1,float16,float16,0,0.08924266695976257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,64,128,1,float16,fp8,0,0.088319996992747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,64,128,1,fp8,fp8,0,0.2019946575164795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,64,0,1,float16,fp8,0,0.0883626639842987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,64,128,1,float16,float16,0,0.06010133524735769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,64,0,1,fp8,fp8,0,0.20145599047342935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,64,0,1,float16,float16,0,0.060005332032839455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,64,128,1,float16,float16,0,0.044405331214269005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,64,128,1,float16,fp8,0,0.057999998331069946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,64,128,1,fp8,fp8,0,0.1172266701857249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,64,0,1,float16,fp8,0,0.057914664347966514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,64,0,1,fp8,fp8,0,0.11574400464693706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,64,0,1,float16,float16,0,0.04465066889921824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,64,128,1,float16,float16,0,0.04513599971930186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,64,128,1,float16,fp8,0,0.04419200122356415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,64,128,1,fp8,fp8,0,0.10743467013041179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,64,0,1,float16,fp8,0,0.04433066646258036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,64,0,1,fp8,fp8,0,0.106495996316274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,64,0,1,float16,float16,0,0.044693330923716225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,64,128,1,float16,fp8,0,0.04456000030040741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,64,128,1,fp8,fp8,0,0.10660800337791443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,64,0,1,float16,fp8,0,0.04478399952252706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,64,0,1,fp8,fp8,0,0.10712533195813496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,64,128,1,float16,float16,0,0.045994664231936135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,64,0,1,float16,float16,0,0.04698666433493296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,64,128,1,float16,fp8,0,0.04602666695912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,64,0,1,float16,float16,0,0.047397335370381675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,64,128,1,float16,fp8,0,0.047295997540156044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,64,128,1,fp8,fp8,0,0.10980266332626343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,64,0,1,float16,fp8,0,0.046341334780057274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,64,0,1,fp8,fp8,0,0.10941333572069804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,64,128,1,float16,float16,0,0.047210668524106346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,64,0,1,float16,float16,0,0.029093332588672638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,64,128,1,fp8,fp8,0,0.11091199517250061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,64,0,1,float16,fp8,0,0.04701333244641622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,64,0,1,fp8,fp8,0,0.1120746632417043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,64,128,1,float16,float16,0,0.030069333811601002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,64,128,1,float16,fp8,0,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,64,0,1,float16,fp8,0,0.028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,64,128,1,fp8,fp8,0,0.06564799944559734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,64,0,1,fp8,fp8,0,0.06423466900984447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,64,128,1,float16,float16,0,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,64,128,1,float16,float16,0,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,64,0,1,float16,float16,0,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,64,128,1,float16,fp8,0,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,64,128,1,float16,fp8,0,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,64,128,1,fp8,fp8,0,0.06218666831652323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,64,0,1,float16,fp8,0,0.027274665733178455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,64,0,1,fp8,fp8,0,0.06150400141874949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,64,0,1,float16,float16,0,0.027349332968393963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,64,128,1,fp8,fp8,0,0.06262933214505513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,64,128,1,fp8,fp8,0,0.06226666768391927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,64,0,1,float16,fp8,0,0.027600000301996868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,64,0,1,fp8,fp8,0,0.061205332477887474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,64,128,1,float16,float16,0,0.028101332485675812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,64,0,1,float16,float16,0,0.028325334191322327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,64,128,1,float16,fp8,0,0.028138667345046997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,64,0,1,float16,fp8,0,0.028959999481836956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,64,0,1,fp8,fp8,0,0.06333333253860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,64,0,1,fp8,fp8,0,0.061792001128196716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,64,128,1,float16,float16,0,0.02810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,64,0,1,float16,float16,0,0.028624000648657482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,64,128,1,float16,fp8,0,0.028330666323502857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,64,128,1,fp8,fp8,0,0.06257066627343495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,64,0,1,float16,fp8,0,0.028277332584063213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,64,128,1,float16,float16,0,0.020768000433842342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,64,0,1,float16,float16,0,0.020917333662509918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,64,128,1,float16,fp8,0,0.02059200033545494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,64,128,1,fp8,fp8,0,0.040634666879971824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,64,0,1,float16,fp8,0,0.02057066683967908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,64,0,1,fp8,fp8,0,0.040522667268911995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,64,128,1,float16,float16,0,0.01942933350801468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,64,0,1,float16,float16,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,64,128,1,float16,fp8,0,0.019845332950353622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,64,128,1,fp8,fp8,0,0.039077334105968475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,64,0,1,float16,fp8,0,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,64,0,1,fp8,fp8,0,0.039120001097520195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,64,128,1,float16,float16,0,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,64,0,1,float16,float16,0,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,64,128,1,float16,float16,0,0.019909333437681198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,64,128,1,float16,fp8,0,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,64,128,1,fp8,fp8,0,0.03892799963553747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,64,0,1,float16,fp8,0,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,64,0,1,fp8,fp8,0,0.03917866696914037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,64,0,1,float16,float16,0,0.020165332903464634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,64,128,1,float16,fp8,0,0.02022933339079221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,64,128,1,fp8,fp8,0,0.03965866565704346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,64,0,1,float16,fp8,0,0.02025066688656807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,64,0,1,fp8,fp8,0,0.03925866633653641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,64,128,1,float16,float16,0,0.020096000283956528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,64,0,1,float16,float16,0,0.019882666567961376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,64,128,1,float16,fp8,0,0.020181333025296528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,64,128,1,fp8,fp8,0,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,64,0,1,float16,fp8,0,0.020234666764736176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,64,0,1,fp8,fp8,0,0.03993066648642222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,64,128,1,float16,float16,0,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,64,0,1,float16,float16,0,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,64,128,1,float16,fp8,0,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,64,128,1,fp8,fp8,0,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,64,0,1,float16,fp8,0,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,64,0,1,fp8,fp8,0,0.02698666602373123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,64,128,1,float16,float16,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,64,0,1,float16,float16,0,0.013829333086808523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,64,128,1,float16,float16,0,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,64,128,1,float16,fp8,0,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,64,128,1,fp8,fp8,0,0.026709333062171936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,64,0,1,float16,fp8,0,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,64,0,1,fp8,fp8,0,0.025786665578683216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,64,0,1,float16,float16,0,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,64,128,1,float16,fp8,0,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,64,128,1,fp8,fp8,0,0.02641066660483678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,64,0,1,float16,fp8,0,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,64,0,1,fp8,fp8,0,0.026154667139053345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,64,128,1,float16,float16,0,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,64,0,1,float16,float16,0,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,64,128,1,float16,fp8,0,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,64,128,1,fp8,fp8,0,0.02645866572856903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,64,0,1,float16,fp8,0,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,64,0,1,fp8,fp8,0,0.02609066665172577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,64,128,1,float16,float16,0,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,64,0,1,float16,float16,0,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,64,128,1,float16,fp8,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,64,128,1,fp8,fp8,0,0.026373334228992462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,64,0,1,float16,fp8,0,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,64,0,1,fp8,fp8,0,0.02621866762638092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,64,128,1,float16,float16,0,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,64,0,1,float16,float16,0,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,64,128,1,float16,fp8,0,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,64,128,1,fp8,fp8,0,0.022613334159056347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,64,0,1,float16,fp8,0,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,64,0,1,fp8,fp8,0,0.022474666436513264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,64,128,1,float16,float16,0,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,64,0,1,fp8,fp8,0,0.02221333235502243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,64,0,1,float16,float16,0,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,64,128,1,float16,fp8,0,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,64,128,1,fp8,fp8,0,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,64,0,1,float16,fp8,0,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,64,128,1,float16,float16,0,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,64,0,1,float16,float16,0,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,64,128,1,fp8,fp8,0,0.021562665700912476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,64,128,1,float16,fp8,0,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,64,0,1,float16,fp8,0,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,64,0,1,fp8,fp8,0,0.02199999988079071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,64,128,1,float16,float16,0,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,64,0,1,float16,float16,0,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,64,128,1,float16,fp8,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,64,128,1,fp8,fp8,0,0.021967999637126923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,64,0,1,float16,fp8,0,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,64,0,1,fp8,fp8,0,0.022416000564893086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,64,128,1,float16,float16,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,64,128,1,float16,float16,0,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,64,0,1,float16,float16,0,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,64,128,1,float16,fp8,0,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,64,128,1,fp8,fp8,0,0.022874665757020313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,64,0,1,float16,fp8,0,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,64,0,1,fp8,fp8,0,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,64,0,1,float16,float16,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,64,128,1,float16,fp8,0,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,64,128,1,fp8,fp8,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,64,0,1,float16,fp8,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,64,0,1,fp8,fp8,0,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,64,128,1,float16,float16,0,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,64,128,1,float16,float16,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,64,0,1,float16,float16,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,64,0,1,float16,float16,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,64,128,1,float16,fp8,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,64,128,1,fp8,fp8,0,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,64,0,1,float16,fp8,0,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,64,0,1,fp8,fp8,0,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,64,128,1,float16,fp8,0,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,64,128,1,fp8,fp8,0,0.01823466643691063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,64,0,1,float16,fp8,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,64,0,1,fp8,fp8,0,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,64,128,1,float16,float16,0,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,64,0,1,float16,float16,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,64,128,1,float16,fp8,0,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,64,0,1,float16,float16,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,64,128,1,fp8,fp8,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,64,0,1,float16,fp8,0,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,64,0,1,fp8,fp8,0,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,64,128,1,float16,float16,0,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,64,128,1,float16,fp8,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,64,128,1,fp8,fp8,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,64,0,1,float16,fp8,0,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,64,0,1,fp8,fp8,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,64,128,1,float16,float16,0,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,64,0,1,float16,float16,0,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,64,128,1,float16,float16,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,64,128,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,64,128,1,fp8,fp8,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,64,0,1,float16,fp8,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,64,0,1,float16,fp8,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,64,0,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,64,0,1,float16,float16,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,64,128,1,float16,fp8,0,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,64,128,1,fp8,fp8,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,64,0,1,fp8,fp8,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,64,128,1,float16,float16,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,64,0,1,fp8,fp8,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,64,128,1,float16,float16,0,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,64,0,1,float16,float16,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,64,128,1,float16,fp8,0,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,64,128,1,fp8,fp8,0,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,64,0,1,float16,fp8,0,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,64,128,1,float16,fp8,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,64,128,1,fp8,fp8,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,64,0,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,64,0,1,fp8,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,64,128,1,float16,float16,0,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,64,0,1,float16,float16,0,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,64,128,1,float16,fp8,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,64,128,1,fp8,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,64,128,1,float16,float16,0,0.11027733484903972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,64,0,1,float16,fp8,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,64,0,1,fp8,fp8,0,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,64,0,1,float16,float16,0,0.10957866907119751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,64,128,1,float16,fp8,0,0.11051199833552043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,64,0,1,float16,fp8,0,0.11010133226712544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,64,0,1,float16,float16,0,0.1109920044740041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,64,128,1,fp8,fp8,0,0.3362826506296794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,64,128,1,float16,float16,0,0.11039466659228007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,64,0,1,fp8,fp8,0,0.337936004002889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,64,128,1,float16,fp8,0,0.10995733737945557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,64,0,1,float16,fp8,0,0.11085333426793416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,64,128,1,fp8,fp8,0,0.3355626662572225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,64,128,1,float16,float16,0,0.11425600449244182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,64,0,1,fp8,fp8,0,0.3351893424987793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,64,0,1,float16,float16,0,0.11357866724332173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,64,128,1,float16,fp8,0,0.1125973363717397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,64,0,1,float16,fp8,0,0.11276800433794658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,64,128,1,fp8,fp8,0,0.3397759993871053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,64,0,1,fp8,fp8,0,0.3390186627705892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,64,128,1,float16,float16,0,0.11497066418329875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,64,128,1,float16,float16,0,0.070783997575442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,64,0,1,float16,float16,0,0.11577066779136658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,64,128,1,float16,fp8,0,0.1134986678759257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,64,128,1,fp8,fp8,0,0.34046932061513263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,64,0,1,float16,fp8,0,0.11412800351778667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,64,0,1,float16,float16,0,0.07147733370463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,64,0,1,fp8,fp8,0,0.34085333347320557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,64,128,1,float16,fp8,0,0.06924800078074138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,64,128,1,fp8,fp8,0,0.18533867597579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,64,0,1,float16,fp8,0,0.06926399966080983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,64,0,1,fp8,fp8,0,0.18502400318781534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,64,0,1,fp8,fp8,0,0.17507733901341757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,64,128,1,fp8,fp8,0,0.1763253410657247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,64,128,1,float16,float16,0,0.05868266522884369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,64,0,1,float16,float16,0,0.05845866600672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,64,128,1,float16,fp8,0,0.05841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,64,128,1,fp8,fp8,0,0.1766293247540792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,64,0,1,float16,fp8,0,0.058917333682378135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,64,128,1,float16,float16,0,0.06075733403364817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,64,128,1,float16,float16,0,0.05899199843406677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,64,0,1,float16,float16,0,0.058917333682378135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,64,128,1,float16,fp8,0,0.05888533095518748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,64,0,1,float16,fp8,0,0.05890666445096334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,64,0,1,fp8,fp8,0,0.17559999227523804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,64,0,1,float16,float16,0,0.06078400214513143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,64,128,1,float16,fp8,0,0.06047999858856201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,64,0,1,float16,float16,0,0.06109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,64,128,1,fp8,fp8,0,0.17894933621088663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,64,0,1,float16,fp8,0,0.06047466893990835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,64,0,1,fp8,fp8,0,0.1789919932683309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,64,128,1,float16,float16,0,0.061535999178886414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,64,128,1,float16,fp8,0,0.061664000153541565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,64,128,1,fp8,fp8,0,0.1795146663983663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,64,0,1,float16,fp8,0,0.06144000093142191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,64,0,1,float16,fp8,0,0.034602666894594826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,64,0,1,fp8,fp8,0,0.17819199959437051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,64,128,1,float16,float16,0,0.0356480007370313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,64,0,1,float16,float16,0,0.035760000348091125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,64,0,1,float16,float16,0,0.033999999364217125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,64,128,1,float16,fp8,0,0.035061334570248924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,64,128,1,fp8,fp8,0,0.09976533055305481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,64,128,1,fp8,fp8,0,0.0960586667060852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,64,128,1,float16,float16,0,0.034117333590984344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,64,0,1,fp8,fp8,0,0.10168000062306722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,64,128,1,float16,fp8,0,0.034559999903043113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,64,0,1,float16,fp8,0,0.03408533334732056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,64,0,1,fp8,fp8,0,0.09664533535639445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,64,128,1,float16,float16,0,0.03420799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,64,0,1,float16,float16,0,0.03477333237727483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,64,128,1,float16,fp8,0,0.03468266626199087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,64,128,1,fp8,fp8,0,0.09685867031415303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,64,0,1,float16,fp8,0,0.034671999514102936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,64,0,1,fp8,fp8,0,0.09637332955996196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,64,128,1,float16,float16,0,0.035391998787721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,64,0,1,float16,float16,0,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,64,128,1,float16,fp8,0,0.03552533437808355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,64,128,1,fp8,fp8,0,0.09757866462071736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,64,128,1,float16,fp8,0,0.03666666646798452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,64,0,1,float16,fp8,0,0.03550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,64,0,1,fp8,fp8,0,0.0976586639881134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,64,128,1,float16,float16,0,0.035029334326585136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,64,0,1,float16,float16,0,0.03554133325815201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,64,128,1,fp8,fp8,0,0.09796266754468282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,64,0,1,float16,fp8,0,0.035418666899204254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,64,0,1,fp8,fp8,0,0.09807999928792317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,64,128,1,float16,float16,0,0.024175999065240223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,64,0,1,float16,float16,0,0.023919999599456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,64,128,1,float16,fp8,0,0.023813332120577495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,64,0,1,float16,float16,0,0.023599999646345775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,64,128,1,float16,float16,0,0.02350933353106181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,64,0,1,float16,fp8,0,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,64,128,1,fp8,fp8,0,0.05858666698137919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,64,0,1,fp8,fp8,0,0.05913066864013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,64,128,1,float16,fp8,0,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,64,128,1,fp8,fp8,0,0.056661332647005715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,64,0,1,float16,fp8,0,0.02378133436044057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,64,0,1,fp8,fp8,0,0.0568800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,64,128,1,float16,float16,0,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,64,0,1,float16,float16,0,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,64,128,1,float16,fp8,0,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,64,128,1,fp8,fp8,0,0.05693866809209188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,64,0,1,float16,fp8,0,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,64,0,1,fp8,fp8,0,0.058687999844551086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,64,128,1,float16,float16,0,0.023930666347344715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,64,128,1,float16,fp8,0,0.02437866727511088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,64,0,1,float16,float16,0,0.023919999599456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,64,128,1,fp8,fp8,0,0.057317331433296204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,64,0,1,float16,fp8,0,0.024485332270463307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,64,0,1,fp8,fp8,0,0.05726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,64,128,1,float16,float16,0,0.02403733382622401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,64,0,1,float16,float16,0,0.023978665471076965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,64,128,1,float16,fp8,0,0.023893333971500397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,64,128,1,fp8,fp8,0,0.05760533114274343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,64,0,1,fp8,fp8,0,0.05809600154558817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,64,0,1,float16,fp8,0,0.024080000817775726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,64,128,1,float16,float16,0,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,64,0,1,float16,float16,0,0.015861333658297855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,64,128,1,fp8,fp8,0,0.036202666660149894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,64,128,1,float16,float16,0,0.015605332950750986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,64,128,1,float16,fp8,0,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,64,0,1,float16,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,64,0,1,fp8,fp8,0,0.035818666219711304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,64,0,1,float16,float16,0,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,64,128,1,float16,fp8,0,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,64,0,1,float16,fp8,0,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,64,128,1,fp8,fp8,0,0.03530666728814443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,64,0,1,fp8,fp8,0,0.03550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,64,128,1,float16,float16,0,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,64,128,1,float16,fp8,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,64,0,1,float16,float16,0,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,64,128,1,fp8,fp8,0,0.035189333061377205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,64,0,1,float16,fp8,0,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,64,0,1,fp8,fp8,0,0.03523733218510946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,64,128,1,float16,float16,0,0.015637333194414776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,64,0,1,float16,float16,0,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,64,0,1,float16,fp8,0,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,64,128,1,float16,fp8,0,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,64,128,1,fp8,fp8,0,0.0352960005402565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,64,128,1,float16,float16,0,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,64,0,1,float16,float16,0,0.015664000064134598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,64,128,1,float16,float16,0,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,64,0,1,fp8,fp8,0,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,64,128,1,float16,fp8,0,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,64,128,1,fp8,fp8,0,0.03527999917666117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,64,0,1,float16,fp8,0,0.016074666132529575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,64,0,1,fp8,fp8,0,0.03569599986076355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,64,0,1,float16,float16,0,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,64,128,1,float16,fp8,0,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,64,128,1,fp8,fp8,0,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,64,0,1,float16,fp8,0,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,64,0,1,fp8,fp8,0,0.025994665920734406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,64,128,1,float16,float16,0,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,64,0,1,float16,float16,0,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,64,128,1,float16,fp8,0,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,64,128,1,fp8,fp8,0,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,64,0,1,float16,fp8,0,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,64,0,1,fp8,fp8,0,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,64,0,1,float16,float16,0,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,64,128,1,float16,float16,0,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,64,128,1,float16,fp8,0,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,64,0,1,float16,float16,0,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,64,128,1,fp8,fp8,0,0.025583999852339428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,64,0,1,float16,fp8,0,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,64,0,1,fp8,fp8,0,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,64,128,1,float16,float16,0,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,64,128,1,float16,fp8,0,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,64,128,1,fp8,fp8,0,0.02638400097688039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,64,0,1,float16,fp8,0,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,64,0,1,fp8,fp8,0,0.02649066597223282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,64,128,1,float16,float16,0,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,64,0,1,float16,float16,0,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,64,128,1,float16,fp8,0,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,64,128,1,fp8,fp8,0,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,64,0,1,float16,fp8,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,64,0,1,fp8,fp8,0,0.025754667818546295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,64,128,1,float16,float16,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,64,0,1,float16,float16,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,64,128,1,float16,fp8,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,64,128,1,fp8,fp8,0,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,64,0,1,float16,fp8,0,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,64,0,1,fp8,fp8,0,0.02248000105222066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,64,128,1,float16,float16,0,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,64,0,1,float16,float16,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,64,128,1,float16,fp8,0,0.012117333710193634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,64,128,1,fp8,fp8,0,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,64,0,1,float16,fp8,0,0.01227733368674914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,64,0,1,fp8,fp8,0,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,64,128,1,float16,float16,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,64,0,1,float16,float16,0,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,64,128,1,float16,fp8,0,0.012245333443085352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,64,128,1,fp8,fp8,0,0.02089066555102666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,64,0,1,float16,fp8,0,0.01251199965675672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,64,0,1,fp8,fp8,0,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,64,128,1,float16,float16,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,64,0,1,float16,float16,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,64,0,1,fp8,fp8,0,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,64,0,1,float16,float16,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,64,128,1,fp8,fp8,0,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,64,128,1,float16,fp8,0,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,64,128,1,float16,fp8,0,0.012362666428089142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,64,0,1,float16,fp8,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,64,128,1,float16,float16,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,64,128,1,fp8,fp8,0,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,64,0,1,float16,fp8,0,0.012432000289360682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,64,0,1,fp8,fp8,0,0.021781332790851593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,64,128,1,float16,float16,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,64,0,1,float16,float16,0,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,64,0,1,float16,float16,0,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,64,128,1,float16,fp8,0,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,64,0,1,float16,fp8,0,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,64,128,1,fp8,fp8,0,0.01807466646035512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,64,0,1,float16,fp8,0,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,64,0,1,fp8,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,64,128,1,float16,float16,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,64,128,1,float16,fp8,0,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,64,128,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,64,0,1,fp8,fp8,0,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,64,128,1,float16,float16,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,64,0,1,float16,float16,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,64,128,1,float16,fp8,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,64,128,1,fp8,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,64,128,1,fp8,fp8,0,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,64,0,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,64,0,1,fp8,fp8,0,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,64,128,1,float16,float16,0,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,64,0,1,float16,float16,0,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,64,128,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,64,0,1,float16,fp8,0,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,64,0,1,fp8,fp8,0,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,64,128,1,float16,float16,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,64,0,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,64,128,1,fp8,fp8,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,64,128,1,float16,fp8,0,0.012080000092585882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,64,0,1,float16,fp8,0,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,64,0,1,fp8,fp8,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,64,128,1,float16,float16,0,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,64,0,1,float16,float16,0,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,64,128,1,float16,fp8,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,64,128,1,fp8,fp8,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,64,0,1,float16,fp8,0,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,64,0,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,64,0,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,64,128,1,float16,float16,0,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,64,0,1,float16,float16,0,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,64,128,1,float16,fp8,0,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,64,128,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,64,0,1,fp8,fp8,0,0.017701332767804463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,64,128,1,float16,float16,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,64,0,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,64,128,1,float16,fp8,0,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,64,128,1,fp8,fp8,0,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,64,0,1,float16,fp8,0,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,64,0,1,fp8,fp8,0,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,64,128,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,64,0,1,float16,float16,0,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,64,128,1,float16,fp8,0,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,64,128,1,fp8,fp8,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,64,0,1,float16,fp8,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,64,0,1,fp8,fp8,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,64,128,1,float16,float16,0,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,64,128,1,float16,fp8,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,64,0,1,float16,float16,0,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,64,0,1,float16,fp8,0,0.012533333152532578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,64,128,1,fp8,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,64,0,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,64,128,1,float16,fp8,0,3.6276426315307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,64,128,1,float16,float16,0,3.6518452962239585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,64,128,1,fp8,fp8,0,4.611642519632976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,64,128,1,float16,float16,0,3.80674680074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,64,128,1,float16,fp8,0,3.776591936747233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,64,128,1,fp8,fp8,0,4.7550506591796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,64,128,1,float16,float16,0,3.8179521560668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,64,128,1,float16,fp8,0,3.779967943827311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,64,0,1,fp8,fp8,0,21.2228266398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,64,0,1,float16,float16,0,23.50941212972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,64,0,1,float16,fp8,0,23.46868896484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,64,128,1,fp8,fp8,0,4.753797213236491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,64,0,1,fp8,fp8,0,21.45886993408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,64,0,1,float16,float16,0,23.817952473958332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,64,128,1,float16,float16,0,3.87064520517985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,64,0,1,float16,fp8,0,23.786768595377605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,64,128,1,float16,fp8,0,3.8163681030273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,64,128,1,fp8,fp8,0,4.80244795481364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,64,0,1,float16,float16,0,23.78123728434245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,64,128,1,float16,float16,0,2.166624069213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,64,128,1,float16,fp8,0,2.1070879300435386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,64,128,1,fp8,fp8,0,2.6863838831583657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,64,0,1,fp8,fp8,0,21.528213500976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,64,0,1,float16,fp8,0,24.24768575032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,64,0,1,float16,float16,0,12.556037902832031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,64,128,1,float16,float16,0,1.7902132670084636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,64,128,1,float16,fp8,0,1.7802880605061848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,64,0,1,float16,float16,0,24.21893310546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,64,0,1,fp8,fp8,0,11.114186604817709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,64,128,1,fp8,fp8,0,2.3279040654500327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,64,0,1,float16,fp8,0,12.885386149088541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,64,0,1,fp8,fp8,0,21.662091573079426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,64,128,1,float16,float16,0,1.8750559488932292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,64,0,1,float16,fp8,0,24.060389200846355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,64,128,1,float16,fp8,0,1.874831994374593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,64,128,1,fp8,fp8,0,2.405423959096273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,64,128,1,float16,float16,0,1.9039200146993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,64,128,1,float16,fp8,0,1.8833759625752766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,64,0,1,float16,float16,0,11.801520029703775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,64,0,1,fp8,fp8,0,11.769643147786459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,64,128,1,fp8,fp8,0,2.433178742726644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,64,0,1,float16,fp8,0,11.943648020426432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,64,0,1,float16,float16,0,11.964640299479166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,64,128,1,float16,float16,0,1.9283626874287922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,64,0,1,fp8,fp8,0,11.455418904622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,64,0,1,float16,fp8,0,11.877707163492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,64,128,1,float16,fp8,0,1.9010292689005535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,64,0,1,float16,float16,0,11.913472493489584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,64,128,1,float16,float16,0,1.0709813435872395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,64,128,1,fp8,fp8,0,2.436367988586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,64,128,1,float16,fp8,0,1.0544959704081218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,64,128,1,fp8,fp8,0,1.3495573997497559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,64,0,1,float16,fp8,0,12.768634796142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,64,0,1,fp8,fp8,0,10.849573771158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,64,0,1,float16,float16,0,6.321157455444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,64,128,1,float16,float16,0,0.9673333168029785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,64,0,1,float16,float16,0,13.26914087931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,64,0,1,fp8,fp8,0,5.595701217651367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,64,0,1,float16,fp8,0,6.379162470499675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,64,128,1,float16,fp8,0,0.9553759892781576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,64,128,1,fp8,fp8,0,1.2314559618632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,64,0,1,fp8,fp8,0,11.161263783772787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,64,128,1,float16,float16,0,0.9710026582082113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,64,128,1,float16,fp8,0,0.9537440141042074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,64,0,1,float16,fp8,0,12.091082255045572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,64,128,1,fp8,fp8,0,1.230463981628418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,64,0,1,float16,float16,0,6.512266794840495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,64,128,1,float16,float16,0,0.981503963470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,64,128,1,float16,fp8,0,0.9922080039978027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,64,0,1,fp8,fp8,0,5.46116828918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,64,128,1,fp8,fp8,0,1.2348586718241374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,64,0,1,float16,fp8,0,5.963738759358724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,64,0,1,float16,float16,0,6.178357442220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,64,128,1,float16,float16,0,0.9764213562011719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,64,0,1,fp8,fp8,0,5.436453501383464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,64,0,1,float16,fp8,0,6.208517074584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,64,128,1,float16,fp8,0,0.9736746946970621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,64,128,1,fp8,fp8,0,1.2483519713083904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,64,128,1,float16,float16,0,0.6856160163879395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,64,0,1,float16,float16,0,6.341253280639648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,64,128,1,float16,fp8,0,0.6901653607686361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,64,128,1,fp8,fp8,0,0.833242654800415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,64,0,1,fp8,fp8,0,5.474645614624023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,64,0,1,float16,fp8,0,6.291605631510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,64,128,1,float16,float16,0,0.6808906396230062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,64,0,1,float16,float16,0,3.22869873046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,64,0,1,float16,float16,0,6.240277608235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,64,128,1,float16,fp8,0,0.6803999741872152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,64,0,1,fp8,fp8,0,5.492693583170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,64,0,1,float16,fp8,0,3.2624588012695312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,64,128,1,fp8,fp8,0,0.8204639752705892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,64,0,1,float16,fp8,0,6.069754918416341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,64,0,1,fp8,fp8,0,2.9748961130777993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,64,128,1,float16,float16,0,0.6854560375213623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,64,128,1,float16,fp8,0,0.685973326365153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,64,0,1,float16,float16,0,3.3155199686686196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,64,128,1,fp8,fp8,0,0.8194879690806071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,64,0,1,float16,fp8,0,3.363055864969889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,64,128,1,float16,float16,0,0.6975253423055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,64,0,1,fp8,fp8,0,2.933135986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,64,0,1,float16,float16,0,3.2805493672688804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,64,128,1,float16,fp8,0,0.6897386709849039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,64,128,1,fp8,fp8,0,0.8274400234222412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,64,0,1,float16,fp8,0,3.3068691889444985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,64,0,1,fp8,fp8,0,2.965903917948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,64,128,1,float16,float16,0,0.6860960324605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,64,0,1,float16,float16,0,3.236133257548014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,64,128,1,float16,fp8,0,0.688368002573649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,64,128,1,fp8,fp8,0,0.8212479750315348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,64,0,1,float16,fp8,0,3.3036959966023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,64,0,1,fp8,fp8,0,2.966597239176432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,64,0,1,float16,float16,0,3.236309369405111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,64,0,1,float16,fp8,0,3.2301225662231445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,64,0,1,fp8,fp8,0,2.9661385218302407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,64,128,1,float16,float16,0,2.724634806315104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,64,128,1,float16,fp8,0,2.7267945607503257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,64,128,1,fp8,fp8,0,3.458085378011068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,64,128,1,float16,float16,0,2.8544692993164062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,64,128,1,float16,fp8,0,2.8419198989868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,64,128,1,fp8,fp8,0,3.587199846903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,64,128,1,float16,float16,0,2.869429270426432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,64,128,1,float16,fp8,0,2.844938596089681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,64,0,1,float16,float16,0,13.735061645507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,64,0,1,fp8,fp8,0,12.472938537597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,64,0,1,float16,fp8,0,13.879440307617188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,64,0,1,float16,float16,0,13.921391805013021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,64,128,1,fp8,fp8,0,3.5629920959472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,64,0,1,fp8,fp8,0,12.58074696858724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,64,0,1,float16,fp8,0,13.847173055013021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,64,128,1,float16,float16,0,2.906240145365397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,64,128,1,float16,fp8,0,2.8490721384684243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,64,0,1,float16,float16,0,13.83792495727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,64,128,1,fp8,fp8,0,3.613312085469564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,64,128,1,float16,float16,0,1.5894346237182617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,64,128,1,float16,fp8,0,1.5575307210286458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,64,128,1,fp8,fp8,0,1.992784023284912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,64,0,1,fp8,fp8,0,12.843007405598959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,64,0,1,float16,fp8,0,14.767450968424479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,64,0,1,float16,float16,0,7.347290674845378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,64,128,1,float16,float16,0,1.355120023091634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,64,128,1,float16,fp8,0,1.3262879848480225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,64,0,1,fp8,fp8,0,6.555925369262695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,64,0,1,float16,fp8,0,7.060453414916992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,64,128,1,fp8,fp8,0,1.7281972567240398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,64,0,1,float16,float16,0,14.996325174967447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,64,0,1,fp8,fp8,0,13.402453104654947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,64,0,1,float16,fp8,0,14.739893595377604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,64,128,1,float16,float16,0,1.3777012825012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,64,128,1,float16,fp8,0,1.355984052022298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,64,128,1,fp8,fp8,0,1.7739787101745605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,64,128,1,float16,float16,0,1.3780694007873535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,64,0,1,float16,float16,0,7.290613174438477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,64,128,1,float16,fp8,0,1.3502240180969238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,64,0,1,fp8,fp8,0,6.260634740193685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,64,128,1,fp8,fp8,0,1.7890559832255046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,64,0,1,float16,fp8,0,7.2680104573567705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,64,0,1,float16,float16,0,7.145109176635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,64,0,1,fp8,fp8,0,6.4030717213948565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,64,128,1,float16,float16,0,1.3948373794555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,64,0,1,float16,fp8,0,7.493994394938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,64,128,1,float16,fp8,0,1.3814239501953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,64,0,1,float16,float16,0,7.211621602376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,64,128,1,fp8,fp8,0,1.8150293032328289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,64,128,1,float16,float16,0,0.8064053058624268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,64,128,1,float16,fp8,0,0.7957973480224609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,64,128,1,fp8,fp8,0,1.0252319971720378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,64,0,1,fp8,fp8,0,6.315077463785808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,64,0,1,float16,fp8,0,6.918778737386067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,64,128,1,float16,float16,0,0.7306773662567139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,64,0,1,float16,float16,0,3.854725201924642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,64,128,1,float16,fp8,0,0.7152213255564371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,64,0,1,float16,float16,0,7.321034749348958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,64,0,1,float16,fp8,0,3.7185281117757163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,64,128,1,fp8,fp8,0,0.9767200152079264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,64,0,1,float16,fp8,0,6.912431716918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,64,0,1,fp8,fp8,0,3.3155199686686196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,64,0,1,fp8,fp8,0,6.352415720621745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,64,0,1,float16,float16,0,3.6347360610961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,64,128,1,float16,float16,0,0.7565920352935791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,64,128,1,float16,fp8,0,0.7258826891581217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,64,128,1,fp8,fp8,0,0.9259573618570963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,64,128,1,float16,float16,0,0.7536373138427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,64,128,1,float16,fp8,0,0.7215147018432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,64,0,1,fp8,fp8,0,3.218677202860514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,64,0,1,float16,fp8,0,3.7995678583780923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,64,128,1,fp8,fp8,0,0.9631786346435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,64,0,1,float16,float16,0,3.6129280726114907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,64,128,1,float16,float16,0,0.8050346374511719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,64,0,1,fp8,fp8,0,3.221429189046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,64,0,1,float16,fp8,0,3.5705385208129883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,64,128,1,float16,fp8,0,0.7507146994272867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,64,0,1,float16,float16,0,3.641866683959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,64,128,1,fp8,fp8,0,0.942080020904541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,64,128,1,float16,float16,0,0.5263893206914266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,64,0,1,fp8,fp8,0,3.225920041402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,64,0,1,float16,fp8,0,3.5392160415649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,64,128,1,float16,fp8,0,0.5208213329315186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,64,0,1,float16,float16,0,3.554101308186849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,64,128,1,fp8,fp8,0,0.6224906841913859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,64,0,1,float16,float16,0,1.9386399586995442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,64,128,1,float16,float16,0,0.5208959976832072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,64,0,1,float16,fp8,0,3.542954762776693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,64,0,1,fp8,fp8,0,3.275178591410319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,64,0,1,float16,fp8,0,1.985413392384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,64,0,1,fp8,fp8,0,1.792784055074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,64,128,1,float16,fp8,0,0.5204266707102457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,64,128,1,fp8,fp8,0,0.625706672668457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,64,0,1,float16,float16,0,1.9166827201843262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,64,128,1,float16,float16,0,0.522874673207601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,64,128,1,float16,fp8,0,0.5197706619898478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,64,128,1,fp8,fp8,0,0.6256746848424276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,64,128,1,float16,float16,0,0.519050677617391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,64,0,1,fp8,fp8,0,1.786351998647054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,64,0,1,float16,fp8,0,1.998261292775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,64,0,1,float16,float16,0,1.9360480308532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,64,128,1,float16,fp8,0,0.5198560158411661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,64,0,1,fp8,fp8,0,1.7883680661519368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,64,0,1,float16,fp8,0,2.0023093223571777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,64,128,1,fp8,fp8,0,0.6255093415578207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,64,128,1,float16,float16,0,0.5235093434651693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,64,0,1,float16,float16,0,1.9698400497436523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,64,128,1,float16,fp8,0,0.5263253450393677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,64,128,1,fp8,fp8,0,0.6250613530476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,64,0,1,float16,fp8,0,1.9681226412455242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,64,0,1,fp8,fp8,0,1.817914644877116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,64,0,1,float16,float16,0,1.9399466514587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,64,0,1,float16,fp8,0,1.961450735727946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,64,0,1,fp8,fp8,0,1.8156639734903972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,64,128,1,float16,float16,0,2.2440479596455893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,64,128,1,float16,fp8,0,2.215013345082601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,64,128,1,fp8,fp8,0,2.9000587463378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,64,128,1,float16,float16,0,2.3763413429260254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,64,128,1,float16,fp8,0,2.3615360260009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,64,128,1,fp8,fp8,0,2.9798666636149087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,64,128,1,float16,float16,0,2.385802745819092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,64,0,1,float16,float16,0,9.993317286173502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,64,0,1,fp8,fp8,0,8.876341501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,64,128,1,float16,fp8,0,2.36734406153361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,64,0,1,float16,fp8,0,9.786373138427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,64,0,1,float16,float16,0,9.908208211263021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,64,0,1,fp8,fp8,0,9.010986963907877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,64,128,1,fp8,fp8,0,2.996058781941732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,64,128,1,float16,float16,0,2.403711954752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,64,0,1,float16,fp8,0,9.889488220214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,64,128,1,float16,fp8,0,2.383552074432373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,64,0,1,float16,float16,0,9.933317184448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,64,128,1,fp8,fp8,0,3.0287679036458335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,64,128,1,float16,float16,0,1.315824031829834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,64,128,1,float16,fp8,0,1.2844586372375488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,64,128,1,fp8,fp8,0,1.6613225936889648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,64,0,1,fp8,fp8,0,8.991680145263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,64,0,1,float16,fp8,0,9.895423889160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,64,128,1,float16,float16,0,1.106384038925171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,64,0,1,float16,float16,0,5.30244255065918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,64,0,1,float16,float16,0,10.027066548665365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,64,128,1,float16,fp8,0,1.0965226491292317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,64,0,1,fp8,fp8,0,9.05294418334961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,64,128,1,fp8,fp8,0,1.5007360776265461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,64,0,1,float16,fp8,0,10.264874776204428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,64,0,1,float16,fp8,0,5.250992139180501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,64,0,1,fp8,fp8,0,4.692053476969401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,64,128,1,float16,float16,0,1.1261013348897297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,64,128,1,float16,fp8,0,1.1200053691864014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,64,0,1,float16,float16,0,5.194394747416179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,64,128,1,fp8,fp8,0,1.488602638244629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,64,128,1,float16,float16,0,1.1310186386108398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,64,0,1,fp8,fp8,0,4.474917411804199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,64,128,1,float16,fp8,0,1.1290132999420166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,64,0,1,float16,fp8,0,5.359466552734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,64,128,1,fp8,fp8,0,1.4662346839904785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,64,0,1,float16,float16,0,5.107162793477376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,64,0,1,fp8,fp8,0,4.492826779683431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,64,128,1,float16,float16,0,1.2109706401824951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,64,0,1,float16,fp8,0,4.961034774780273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,64,128,1,float16,fp8,0,1.1779680252075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,64,128,1,fp8,fp8,0,1.4957547187805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,64,0,1,float16,float16,0,5.16977055867513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,64,128,1,float16,float16,0,0.6907467047373453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,64,0,1,fp8,fp8,0,4.502976099650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,64,128,1,float16,fp8,0,0.7098293304443359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,64,0,1,float16,fp8,0,5.050565401713054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,64,128,1,fp8,fp8,0,0.8616586526234945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,64,128,1,float16,float16,0,0.6121333440144857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,64,0,1,float16,float16,0,2.616586685180664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,64,0,1,float16,float16,0,5.366986592610677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,64,0,1,fp8,fp8,0,4.626517295837402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,64,128,1,float16,fp8,0,0.6058026552200317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,64,128,1,fp8,fp8,0,0.8377066453297933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,64,0,1,float16,fp8,0,5.5262025197347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,64,0,1,fp8,fp8,0,2.5055200258890786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,64,0,1,float16,fp8,0,2.795813242594401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,64,128,1,float16,float16,0,0.614031990369161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,64,0,1,float16,float16,0,2.5275039672851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,64,128,1,float16,fp8,0,0.621120015780131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,64,128,1,fp8,fp8,0,0.82204270362854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,64,0,1,float16,fp8,0,2.5180106163024902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,64,128,1,float16,float16,0,0.6213653484980265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,64,0,1,fp8,fp8,0,2.3610612551371255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,64,0,1,float16,float16,0,2.62882137298584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,64,128,1,float16,fp8,0,0.6173866589864095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,64,0,1,float16,fp8,0,2.541621367136637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,64,128,1,fp8,fp8,0,0.7931466897328695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,64,0,1,fp8,fp8,0,2.361471970876058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,64,128,1,float16,float16,0,0.6268800099690756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,64,0,1,float16,float16,0,2.532810688018799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,64,128,1,float16,fp8,0,0.62062935034434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,64,128,1,fp8,fp8,0,0.7993760108947754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,64,0,1,float16,fp8,0,2.565429369608561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,64,128,1,float16,float16,0,0.4385439952214559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,64,0,1,fp8,fp8,0,2.3208959897359214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,64,128,1,float16,fp8,0,0.4405279954274495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,64,0,1,float16,float16,0,2.5578667322794595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,64,128,1,fp8,fp8,0,0.528495987256368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,64,0,1,float16,float16,0,1.411520004272461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,64,0,1,fp8,fp8,0,2.361599922180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,64,128,1,float16,float16,0,0.4396959940592448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,64,0,1,float16,fp8,0,2.5352320671081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,64,128,1,float16,fp8,0,0.4379306634267171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,64,0,1,float16,fp8,0,1.4119839668273926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,64,0,1,fp8,fp8,0,1.2697386741638184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,64,128,1,fp8,fp8,0,0.5301386515299479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,64,0,1,float16,float16,0,1.4151733716328938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,64,128,1,float16,float16,0,0.44090668360392254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,64,0,1,float16,fp8,0,1.41539732615153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,64,0,1,fp8,fp8,0,1.2861599922180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,64,128,1,float16,fp8,0,0.44039467970530194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,64,128,1,fp8,fp8,0,0.5305066506067911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,64,0,1,float16,float16,0,1.4329066276550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,64,128,1,float16,float16,0,0.43832000096638996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,64,128,1,float16,fp8,0,0.4412906567255656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,64,128,1,fp8,fp8,0,0.5273386637369791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,64,0,1,fp8,fp8,0,1.2768853505452473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,64,0,1,float16,fp8,0,1.4362400372823079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,64,128,1,float16,float16,0,0.43853334585825604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,64,0,1,float16,float16,0,1.4275892575581868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,64,128,1,float16,fp8,0,0.4416373173395793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,64,128,1,fp8,fp8,0,0.5320053497950236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,64,0,1,fp8,fp8,0,1.2593386967976887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,64,0,1,float16,fp8,0,1.437162717183431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,64,0,1,float16,float16,0,1.4162559509277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,64,0,1,fp8,fp8,0,1.2603840033213298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,64,0,1,float16,fp8,0,1.4373386700948079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,64,128,1,float16,float16,0,3.621317227681478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,64,128,1,float16,fp8,0,3.5834665298461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,64,128,1,fp8,fp8,0,4.518501281738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,64,128,1,float16,float16,0,3.76908270517985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,64,128,1,float16,fp8,0,3.7456000645955405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,64,128,1,fp8,fp8,0,4.7117493947347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,64,128,1,float16,float16,0,3.765061378479004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,64,0,1,fp8,fp8,0,11.811594645182291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,64,0,1,float16,float16,0,13.102415720621744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,64,0,1,float16,fp8,0,13.015684763590494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,64,0,1,float16,float16,0,13.13918940226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,64,128,1,float16,fp8,0,3.732341448465983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,64,0,1,fp8,fp8,0,11.962443033854166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,64,128,1,fp8,fp8,0,4.733008066813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,64,0,1,float16,fp8,0,13.036000569661459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,64,128,1,float16,float16,0,3.8153279622395835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,64,128,1,float16,fp8,0,3.79254945119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,64,0,1,float16,float16,0,13.135088602701822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,64,128,1,fp8,fp8,0,4.790570576985677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,64,128,1,float16,float16,0,2.128906726837158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,64,128,1,float16,fp8,0,2.081280072530111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,64,0,1,fp8,fp8,0,12.298239390055338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,64,0,1,float16,fp8,0,13.250554402669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,64,128,1,fp8,fp8,0,2.6377280553181968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,64,128,1,float16,float16,0,1.753109296162923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,64,0,1,float16,float16,0,6.8466135660807295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,64,0,1,float16,float16,0,13.285045623779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,64,128,1,float16,fp8,0,1.7063040733337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,64,0,1,fp8,fp8,0,12.668975830078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,64,0,1,float16,fp8,0,13.995653788248697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,64,0,1,float16,fp8,0,7.054751714070638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,64,0,1,fp8,fp8,0,6.815039952596028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,64,128,1,fp8,fp8,0,2.3237600326538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,64,128,1,float16,float16,0,1.849621295928955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,64,0,1,float16,float16,0,6.695503870646159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,64,128,1,float16,fp8,0,1.8419200579325359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,64,128,1,fp8,fp8,0,2.3786400159200034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,64,128,1,float16,float16,0,1.8667200406392415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,64,0,1,float16,fp8,0,6.771002451578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,64,0,1,fp8,fp8,0,6.168501536051433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,64,128,1,float16,fp8,0,1.841551939646403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,64,128,1,fp8,fp8,0,2.3864906628926597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,64,0,1,float16,fp8,0,6.768831888834636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,64,0,1,fp8,fp8,0,6.026490529378255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,64,0,1,float16,float16,0,7.275920232137044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,64,128,1,float16,float16,0,1.8863040606180828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,64,0,1,float16,float16,0,6.618991851806641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,64,128,1,float16,fp8,0,1.9908587137858074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,64,128,1,fp8,fp8,0,2.4226667086283364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,64,128,1,float16,float16,0,1.0443039735158284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,64,0,1,fp8,fp8,0,6.027834574381511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,64,0,1,float16,fp8,0,7.110720316569011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,64,128,1,float16,fp8,0,1.0172213713328044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,64,128,1,fp8,fp8,0,1.303008000055949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,64,0,1,float16,float16,0,6.648949305216472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,64,0,1,float16,float16,0,3.5368639628092446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,64,128,1,float16,float16,0,0.8907732963562012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,64,0,1,float16,fp8,0,6.670725504557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,64,128,1,float16,fp8,0,0.9137919743855795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,64,0,1,fp8,fp8,0,6.06271489461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,64,128,1,fp8,fp8,0,1.1503360271453857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,64,0,1,fp8,fp8,0,3.145146687825521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,64,0,1,float16,fp8,0,3.6766134897867837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,64,0,1,float16,float16,0,3.3730185826619468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,64,128,1,float16,float16,0,0.9065813223520914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,64,128,1,float16,fp8,0,0.8935840129852295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,64,128,1,fp8,fp8,0,1.1699573198954265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,64,0,1,float16,fp8,0,3.2952534357706704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,64,128,1,float16,float16,0,0.9116319815317789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,64,0,1,fp8,fp8,0,2.9927199681599936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,64,128,1,float16,fp8,0,0.8980533281962076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,64,128,1,fp8,fp8,0,1.1770506699879963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,64,0,1,float16,float16,0,3.373776117960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,64,0,1,fp8,fp8,0,2.9979092280069985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,64,128,1,float16,float16,0,0.9216372966766357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,64,0,1,float16,fp8,0,3.3383572896321616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,64,0,1,float16,float16,0,3.4560747146606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,64,128,1,float16,fp8,0,0.9331573645273844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,64,128,1,fp8,fp8,0,1.1871679623921711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,64,128,1,float16,float16,0,0.562122662862142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,64,0,1,float16,fp8,0,3.387669245402018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,64,0,1,fp8,fp8,0,3.0042241414388022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,64,128,1,float16,fp8,0,0.5422559976577759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,64,128,1,fp8,fp8,0,0.7132053375244141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,64,0,1,float16,float16,0,1.7363039652506511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,64,0,1,float16,float16,0,3.4790932337443032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,64,128,1,float16,float16,0,0.5070986747741699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,64,0,1,float16,fp8,0,3.3722238540649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,64,0,1,fp8,fp8,0,3.0629119873046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,64,128,1,float16,fp8,0,0.4899839957555135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,64,128,1,fp8,fp8,0,0.6553599834442139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,64,0,1,fp8,fp8,0,1.6020372708638508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,64,0,1,float16,fp8,0,1.7866826057434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,64,0,1,float16,float16,0,1.6955359776814778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,64,128,1,float16,float16,0,0.4980693260828654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,64,128,1,float16,fp8,0,0.4915039936701457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,64,128,1,fp8,fp8,0,0.6279040177663168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,64,0,1,float16,fp8,0,1.7203307151794434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,64,0,1,fp8,fp8,0,1.596677303314209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,64,128,1,float16,float16,0,0.49980799357096356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,64,0,1,float16,float16,0,1.7261494000752766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,64,128,1,float16,fp8,0,0.49004801114400226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,64,0,1,float16,fp8,0,1.7173439661661785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,64,128,1,fp8,fp8,0,0.642192006111145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,64,0,1,fp8,fp8,0,1.5641813278198242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,64,128,1,float16,float16,0,0.5149386723836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,64,0,1,float16,float16,0,1.6971093813578289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,64,128,1,float16,fp8,0,0.5090453227361044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,64,0,1,float16,fp8,0,1.6939412752787273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,64,128,1,fp8,fp8,0,0.6476106643676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,64,0,1,fp8,fp8,0,1.5670453707377117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,64,0,1,float16,float16,0,1.7002506256103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,64,128,1,float16,float16,0,0.35807998975118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,64,128,1,float16,fp8,0,0.3567306598027547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,64,128,1,fp8,fp8,0,0.43054401874542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,64,0,1,float16,float16,0,0.9844000339508057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,64,128,1,float16,float16,0,0.3571999867757161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,64,0,1,float16,fp8,0,0.9707466761271158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,64,128,1,float16,fp8,0,0.3556319872538249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,64,0,1,fp8,fp8,0,1.5764907201131184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,64,0,1,fp8,fp8,0,0.8655146757761637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,64,128,1,fp8,fp8,0,0.43308266003926593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,64,0,1,float16,fp8,0,1.697173277537028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,64,0,1,float16,float16,0,0.9716266791025797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,64,128,1,float16,float16,0,0.355679988861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,64,0,1,float16,fp8,0,0.9807306925455729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,64,0,1,fp8,fp8,0,0.861957311630249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,64,128,1,float16,fp8,0,0.35757867495218915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,64,128,1,fp8,fp8,0,0.4332746664683024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,64,0,1,float16,float16,0,0.972762664159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,64,128,1,float16,float16,0,0.3556426763534546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,64,0,1,float16,fp8,0,0.971232016881307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,64,128,1,float16,fp8,0,0.35841067632039386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,64,0,1,fp8,fp8,0,0.8743147055308024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,64,128,1,fp8,fp8,0,0.43281598885854083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,64,128,1,float16,float16,0,0.3561546802520752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,64,0,1,float16,float16,0,0.9731520016988119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,64,128,1,float16,fp8,0,0.35676801204681396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,64,0,1,float16,float16,0,0.9734400113423666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,64,0,1,fp8,fp8,0,0.8767413298288981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,64,0,1,float16,fp8,0,0.9876213073730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,64,128,1,fp8,fp8,0,0.43332799275716144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,64,0,1,float16,fp8,0,0.9885280132293701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,64,0,1,fp8,fp8,0,0.8631839752197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,64,128,1,float16,float16,0,2.6892639795939126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,64,128,1,float16,fp8,0,2.701477368672689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,64,128,1,float16,float16,0,2.826863924662272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,64,128,1,fp8,fp8,0,3.4242932001749673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,64,128,1,float16,fp8,0,2.803237279256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,64,128,1,fp8,fp8,0,3.5353867212931314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,64,0,1,fp8,fp8,0,7.068656285603841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,64,0,1,float16,float16,0,7.774096171061198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,64,0,1,float16,fp8,0,7.752597173055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,64,0,1,float16,float16,0,7.902432123819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,64,128,1,float16,float16,0,2.8335679372151694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,64,128,1,float16,fp8,0,2.8043200174967446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,64,0,1,float16,fp8,0,8.051477432250977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,64,128,1,fp8,fp8,0,3.553584098815918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,64,0,1,fp8,fp8,0,7.199717203776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,64,128,1,float16,float16,0,2.8633012771606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,64,128,1,float16,fp8,0,2.8537333806355796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,64,0,1,float16,float16,0,7.903968175252278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,64,128,1,fp8,fp8,0,3.594794591267904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,64,0,1,float16,fp8,0,7.95683224995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,64,128,1,float16,float16,0,1.5849439303080242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,64,0,1,fp8,fp8,0,7.209450403849284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,64,128,1,float16,fp8,0,1.5414400100708008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,64,128,1,fp8,fp8,0,1.9843467076619465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,64,128,1,float16,float16,0,1.2982239723205566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,64,0,1,float16,float16,0,7.984879811604817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,64,0,1,float16,float16,0,4.2166134516398115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,64,128,1,float16,fp8,0,1.2801813284556072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,64,0,1,fp8,fp8,0,7.264074961344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,64,0,1,float16,fp8,0,4.14790407816569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,64,0,1,fp8,fp8,0,3.8249918619791665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,64,0,1,float16,fp8,0,7.87660280863444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,64,128,1,fp8,fp8,0,1.685109297434489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,64,128,1,float16,float16,0,1.311557372411092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,64,0,1,float16,float16,0,3.8977600733439126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,64,0,1,float16,fp8,0,3.9348319371541343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,64,128,1,float16,fp8,0,1.2968693574269612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,64,0,1,fp8,fp8,0,3.5074132283528647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,64,128,1,fp8,fp8,0,1.7235786120096843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,64,128,1,float16,float16,0,1.3794453938802083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,64,0,1,float16,float16,0,3.8643786112467446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,64,128,1,float16,fp8,0,1.3066933155059814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,64,128,1,fp8,fp8,0,1.7409653663635254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,64,0,1,fp8,fp8,0,3.563269297281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,64,0,1,float16,fp8,0,3.9243361155192056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,64,128,1,float16,float16,0,1.35807466506958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,64,0,1,float16,float16,0,3.886325200398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,64,128,1,float16,fp8,0,1.336133321126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,64,0,1,fp8,fp8,0,3.5815200805664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,64,0,1,float16,fp8,0,4.034677187601726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,64,128,1,float16,float16,0,0.7822879950205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,64,128,1,fp8,fp8,0,1.7594666481018066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,64,0,1,float16,float16,0,3.984304110209147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,64,128,1,float16,fp8,0,0.7670880158742269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,64,128,1,fp8,fp8,0,1.0153333346048992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,64,0,1,float16,float16,0,2.1659199396769204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,64,0,1,fp8,fp8,0,1.8937439918518066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,64,128,1,float16,float16,0,0.7177920341491699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,64,0,1,fp8,fp8,0,3.607669194539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,64,0,1,float16,fp8,0,2.098362604777018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,64,0,1,float16,fp8,0,3.8721278508504233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,64,128,1,float16,fp8,0,0.69706130027771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,64,128,1,fp8,fp8,0,0.9116480350494385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,64,0,1,float16,float16,0,1.9630452791849773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,64,128,1,float16,fp8,0,0.6737386385599772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,64,128,1,float16,float16,0,0.6755680243174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,64,0,1,float16,float16,0,1.9675572713216145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,64,128,1,fp8,fp8,0,0.890554666519165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,64,0,1,fp8,fp8,0,1.8612693150838215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,64,0,1,float16,fp8,0,1.9765440622965496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,64,128,1,float16,fp8,0,0.6798559824625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,64,128,1,float16,float16,0,0.6875627040863037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,64,0,1,float16,fp8,0,1.959536075592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,64,0,1,fp8,fp8,0,1.8070665995279949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,64,128,1,fp8,fp8,0,0.8944746653238932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,64,0,1,float16,float16,0,1.9702666600545247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,64,128,1,float16,float16,0,0.6959786415100098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,64,128,1,float16,fp8,0,0.6889920234680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,64,128,1,fp8,fp8,0,0.9163253307342529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,64,128,1,float16,float16,0,0.41152532895406085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,64,0,1,float16,fp8,0,1.9621866544087727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,64,0,1,fp8,fp8,0,1.8220586776733398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,64,0,1,float16,float16,0,1.9779574076334636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,64,128,1,float16,fp8,0,0.42239999771118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,64,128,1,fp8,fp8,0,0.5405973196029663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,64,0,1,float16,float16,0,1.0618826548258464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,64,128,1,float16,float16,0,0.38016533851623535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,64,0,1,float16,fp8,0,1.9736639658610027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,64,0,1,fp8,fp8,0,1.8321973482767742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,64,128,1,float16,fp8,0,0.38124267260233563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,64,0,1,float16,fp8,0,1.0656800270080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,64,0,1,fp8,fp8,0,0.9981119632720947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,64,128,1,fp8,fp8,0,0.4848586718241374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,64,0,1,float16,float16,0,1.0516746838887532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,64,128,1,float16,float16,0,0.387935996055603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,64,0,1,float16,fp8,0,1.0451520284016926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,64,128,1,fp8,fp8,0,0.4891573190689087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,64,0,1,fp8,fp8,0,0.9531146685282389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,64,128,1,float16,fp8,0,0.3775893449783325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,64,0,1,float16,float16,0,1.0518240133921306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,64,128,1,float16,float16,0,0.3893493413925171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,64,0,1,float16,fp8,0,1.0257813135782878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,64,128,1,float16,fp8,0,0.3778080145517985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,64,0,1,fp8,fp8,0,0.9741120338439941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,64,128,1,fp8,fp8,0,0.4889119863510132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,64,0,1,float16,float16,0,1.0567733446757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,64,128,1,float16,float16,0,0.38758401075998944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,64,0,1,float16,fp8,0,1.0303200085957844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,64,0,1,fp8,fp8,0,0.9771040280659994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,64,128,1,float16,fp8,0,0.3830346663792928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,64,0,1,float16,float16,0,1.0530613263448079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,64,128,1,fp8,fp8,0,0.49196799596150714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,64,0,1,float16,fp8,0,1.0342346827189128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,64,128,1,float16,float16,0,0.27370667457580566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,64,0,1,float16,float16,0,0.6195679903030396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,64,0,1,fp8,fp8,0,0.9658346970876058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,64,128,1,float16,fp8,0,0.2749493320782979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,64,128,1,fp8,fp8,0,0.33448533217112225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,64,0,1,float16,fp8,0,0.6118613481521606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,64,128,1,float16,float16,0,0.2714773416519165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,64,128,1,float16,fp8,0,0.2722613414128621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,64,0,1,fp8,fp8,0,0.5414453347524008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,64,0,1,float16,float16,0,0.6122026840845743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,64,128,1,fp8,fp8,0,0.331061323483785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,64,128,1,float16,float16,0,0.27138666311899823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,64,0,1,float16,fp8,0,0.614954670270284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,64,0,1,fp8,fp8,0,0.5458826621373495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,64,128,1,float16,fp8,0,0.27447466055552167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,64,0,1,float16,float16,0,0.6122186581293741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,64,128,1,fp8,fp8,0,0.33234665791193646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,64,0,1,float16,fp8,0,0.6136373281478882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,64,128,1,float16,float16,0,0.27357866366704303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,64,0,1,fp8,fp8,0,0.5415680011113485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,64,128,1,float16,fp8,0,0.2749279936154683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,64,0,1,float16,float16,0,0.6163839896519979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,64,128,1,fp8,fp8,0,0.33128533760706586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,64,128,1,float16,float16,0,0.2736053268114726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,64,0,1,float16,fp8,0,0.6116533279418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,64,0,1,fp8,fp8,0,0.5420106649398804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,64,128,1,float16,fp8,0,0.27451733748118085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,64,0,1,float16,float16,0,0.6124373277028402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,64,128,1,fp8,fp8,0,0.3334133227666219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,64,0,1,float16,fp8,0,0.6137706836064657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,64,0,1,fp8,fp8,0,0.5419093370437622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,64,128,1,float16,float16,0,3.587552070617676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,64,128,1,float16,fp8,0,3.5656000773111978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,64,128,1,float16,float16,0,3.736698786417643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,64,128,1,fp8,fp8,0,4.520037333170573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,64,128,1,float16,fp8,0,3.6991093953450522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,64,0,1,float16,float16,0,7.7249495188395185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,64,0,1,fp8,fp8,0,7.0059254964192705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,64,0,1,float16,fp8,0,7.641424179077148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,64,128,1,fp8,fp8,0,4.661989212036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,64,0,1,float16,float16,0,7.905029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,64,128,1,float16,float16,0,3.7289600372314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,64,0,1,fp8,fp8,0,7.13804817199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,64,128,1,float16,fp8,0,3.700826644897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,64,0,1,float16,fp8,0,7.781194686889648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,64,128,1,fp8,fp8,0,4.667562802632649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,64,0,1,float16,float16,0,7.827541351318359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,64,128,1,float16,float16,0,3.7865705490112305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,64,128,1,float16,fp8,0,3.7505919138590493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,64,0,1,fp8,fp8,0,7.204837163289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,64,0,1,float16,fp8,0,7.797072092692058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,64,128,1,float16,float16,0,2.1124960581461587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,64,128,1,fp8,fp8,0,4.728394508361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,64,0,1,float16,float16,0,8.014570871988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,64,128,1,float16,fp8,0,2.0630240440368652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,64,0,1,float16,float16,0,4.225157419840495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,64,128,1,fp8,fp8,0,2.627247969309489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,64,0,1,float16,fp8,0,7.9194081624348955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,64,128,1,float16,float16,0,1.7638826370239258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,64,0,1,float16,fp8,0,4.282032012939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,64,0,1,fp8,fp8,0,3.870037396748861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,64,128,1,float16,fp8,0,1.7267146110534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,64,0,1,fp8,fp8,0,7.2634932200113935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,64,128,1,fp8,fp8,0,2.236069361368815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,64,0,1,float16,float16,0,4.024335861206055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,64,128,1,float16,float16,0,1.828111966451009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,64,128,1,float16,fp8,0,1.8204906781514485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,64,0,1,fp8,fp8,0,3.500138600667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,64,0,1,float16,fp8,0,3.9406827290852866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,64,128,1,fp8,fp8,0,2.331770737965902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,64,0,1,float16,float16,0,4.027578671773274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,64,128,1,float16,float16,0,1.8431199391682942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,64,0,1,float16,fp8,0,3.869445482889811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,64,128,1,float16,fp8,0,1.8281280199686687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,64,0,1,fp8,fp8,0,3.608544031778971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,64,128,1,fp8,fp8,0,2.3649919827779136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,64,0,1,float16,float16,0,3.916816075642904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,64,128,1,float16,float16,0,1.87446928024292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,64,128,1,float16,fp8,0,1.8547946612040203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,64,0,1,float16,fp8,0,4.125941276550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,64,0,1,fp8,fp8,0,3.622133255004883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,64,128,1,float16,float16,0,1.0302986303965251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,64,128,1,fp8,fp8,0,2.390490690867106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,64,0,1,float16,float16,0,3.9189494450887046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,64,128,1,float16,fp8,0,1.0217866897583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,64,0,1,float16,float16,0,2.060266653696696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,64,0,1,float16,fp8,0,4.019114812215169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,64,0,1,fp8,fp8,0,3.624042510986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,64,128,1,fp8,fp8,0,1.2848320007324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,64,128,1,float16,float16,0,0.8716959953308105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,64,128,1,float16,fp8,0,0.881498654683431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,64,0,1,float16,fp8,0,2.0282133420308432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,64,128,1,fp8,fp8,0,1.142858664194743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,64,0,1,fp8,fp8,0,1.9187466303507488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,64,0,1,float16,float16,0,1.9049760500590007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,64,128,1,float16,float16,0,0.8935573101043701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,64,0,1,float16,fp8,0,1.9232479731241863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,64,128,1,float16,fp8,0,0.8641119798024496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,64,0,1,fp8,fp8,0,1.7569386164347331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,64,128,1,fp8,fp8,0,1.1359573205312092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,64,0,1,float16,float16,0,1.9130880037943523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,64,128,1,float16,float16,0,0.9002026716868082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,64,128,1,float16,fp8,0,0.87009596824646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,64,0,1,float16,fp8,0,1.8981866836547852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,64,0,1,fp8,fp8,0,1.7669226328531902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,64,128,1,fp8,fp8,0,1.1481599807739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,64,0,1,float16,float16,0,1.91320006052653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,64,128,1,float16,float16,0,0.8934186299641927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,64,128,1,float16,fp8,0,0.8852799733479818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,64,0,1,float16,fp8,0,1.8985333442687988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,64,0,1,fp8,fp8,0,1.8009173075358074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,64,128,1,fp8,fp8,0,1.161679983139038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,64,128,1,float16,float16,0,0.5261760155359904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,64,0,1,float16,float16,0,1.9259093602498372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,64,128,1,float16,fp8,0,0.5251839955647787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,64,0,1,float16,fp8,0,1.9112745920817058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,64,0,1,float16,float16,0,1.045093297958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,64,128,1,fp8,fp8,0,0.6645973523457845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,64,128,1,float16,float16,0,0.4718773365020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,64,0,1,fp8,fp8,0,1.7807520230611165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,64,0,1,float16,fp8,0,1.0380213260650635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,64,0,1,fp8,fp8,0,0.9886133670806885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,64,128,1,float16,fp8,0,0.45442132155100506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,64,128,1,fp8,fp8,0,0.6014560063680013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,64,0,1,float16,float16,0,0.9870879650115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,64,0,1,float16,fp8,0,0.9817813237508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,64,128,1,float16,float16,0,0.46618131796518963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,64,0,1,fp8,fp8,0,0.9117759863535563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,64,128,1,float16,fp8,0,0.4675360123316447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,64,128,1,fp8,fp8,0,0.6100000143051147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,64,0,1,float16,float16,0,1.0046772956848145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,64,0,1,float16,fp8,0,0.9681920210520426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,64,128,1,float16,float16,0,0.468389352162679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,64,0,1,fp8,fp8,0,0.9179946581522623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,64,128,1,float16,fp8,0,0.46742932001749676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,64,128,1,fp8,fp8,0,0.612549344698588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,64,0,1,float16,float16,0,0.9911839962005615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,64,128,1,float16,float16,0,0.4724533160527547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,64,128,1,float16,fp8,0,0.4728320042292277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,64,0,1,float16,fp8,0,0.9892906347910563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,64,0,1,fp8,fp8,0,0.9113492965698242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,64,128,1,fp8,fp8,0,0.6189813216527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,64,0,1,float16,float16,0,0.9960853258768717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,64,128,1,float16,float16,0,0.28593599796295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,64,128,1,float16,fp8,0,0.2866080005963643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,64,0,1,float16,float16,0,0.5568693478902181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,64,128,1,fp8,fp8,0,0.36106133460998535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,64,128,1,float16,float16,0,0.2632853388786316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,64,0,1,float16,fp8,0,1.003941297531128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,64,0,1,fp8,fp8,0,0.9232906500498453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,64,0,1,float16,fp8,0,0.5627946853637695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,64,0,1,fp8,fp8,0,0.5018879969914755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,64,128,1,float16,fp8,0,0.2606933315594991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,64,0,1,float16,float16,0,0.5322826703389486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,64,128,1,fp8,fp8,0,0.33195199569066364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,64,0,1,float16,fp8,0,0.530847986539205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,64,128,1,float16,float16,0,0.25964800516764325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,64,0,1,fp8,fp8,0,0.4806079864501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,64,128,1,float16,fp8,0,0.2576533357302348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,64,0,1,float16,float16,0,0.5392160018285116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,64,128,1,fp8,fp8,0,0.33858664830525714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,64,0,1,float16,fp8,0,0.5356800158818563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,64,0,1,fp8,fp8,0,0.4705653190612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,64,128,1,float16,float16,0,0.26424533128738403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,64,128,1,float16,fp8,0,0.2613706588745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,64,0,1,float16,float16,0,0.5313173135121664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,64,128,1,fp8,fp8,0,0.33644266923268634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,64,0,1,float16,fp8,0,0.5370560089747111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,64,128,1,float16,float16,0,0.2686293323834737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,64,0,1,fp8,fp8,0,0.47747735182444256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,64,128,1,float16,fp8,0,0.2634399930636088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,64,0,1,float16,float16,0,0.5462293227513632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,64,128,1,fp8,fp8,0,0.3424373467763265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,64,128,1,float16,float16,0,0.19171200195948282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,64,0,1,float16,fp8,0,0.5330080191294352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,64,0,1,fp8,fp8,0,0.4780426820119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,64,0,1,float16,float16,0,0.3161333401997884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,64,128,1,float16,fp8,0,0.19149333238601685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,64,128,1,fp8,fp8,0,0.23466134071350098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,64,0,1,float16,fp8,0,0.317466676235199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,64,128,1,float16,float16,0,0.18950400749842325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,64,0,1,fp8,fp8,0,0.29473066329956055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,64,128,1,float16,fp8,0,0.1896160046259562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,64,0,1,float16,float16,0,0.31085866689682007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,64,128,1,fp8,fp8,0,0.23240532477696738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,64,0,1,float16,fp8,0,0.3130613366762797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,64,0,1,fp8,fp8,0,0.2937279939651489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,64,128,1,float16,float16,0,0.18968000014623007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,64,0,1,float16,float16,0,0.3122933308283488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,64,128,1,float16,fp8,0,0.1895893414815267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,64,128,1,fp8,fp8,0,0.23215999205907187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,64,0,1,float16,fp8,0,0.31190399328867596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,64,0,1,fp8,fp8,0,0.2953439950942993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,64,128,1,float16,float16,0,0.18920000394185385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,64,0,1,float16,float16,0,0.31198400259017944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,64,128,1,float16,fp8,0,0.18997333447138467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,64,128,1,fp8,fp8,0,0.2315679987271627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,64,0,1,float16,fp8,0,0.31271467606226605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,64,0,1,fp8,fp8,0,0.29360532760620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,64,128,1,float16,float16,0,0.19160000483194986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,64,0,1,float16,float16,0,0.31245867411295575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,64,128,1,float16,fp8,0,0.19057599703470865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,64,128,1,fp8,fp8,0,0.2334559957186381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,64,0,1,float16,fp8,0,0.31308799982070923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,64,0,1,fp8,fp8,0,0.294320007165273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,64,128,1,float16,float16,0,2.6207146644592285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,64,128,1,float16,fp8,0,2.6014720598856607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,64,128,1,float16,float16,0,2.701989491780599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,64,128,1,fp8,fp8,0,3.3460480372111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,64,0,1,float16,float16,0,4.7775678634643555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,64,0,1,fp8,fp8,0,4.339365323384603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,64,0,1,float16,fp8,0,4.690245310465495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,64,128,1,float16,fp8,0,2.6861546834309897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,64,128,1,fp8,fp8,0,3.394554773966471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,64,0,1,float16,float16,0,4.796890576680501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,64,128,1,float16,float16,0,2.7211519877115884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,64,128,1,float16,fp8,0,2.7023681004842124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,64,0,1,float16,fp8,0,4.783328056335449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,64,0,1,fp8,fp8,0,4.383461316426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,64,0,1,float16,float16,0,4.94645881652832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,64,128,1,fp8,fp8,0,3.492293357849121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,64,128,1,float16,float16,0,2.758053461710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,64,128,1,float16,fp8,0,2.731013298034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,64,0,1,fp8,fp8,0,4.43342399597168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,64,0,1,float16,fp8,0,4.8873545328776045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,64,128,1,fp8,fp8,0,3.496687889099121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,64,128,1,float16,float16,0,1.5587199529012044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,64,0,1,float16,float16,0,4.8653866449991865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,64,128,1,float16,fp8,0,1.51200532913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,64,128,1,fp8,fp8,0,1.9521600405375164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,64,0,1,fp8,fp8,0,4.51969051361084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,64,0,1,float16,float16,0,2.6172053019205728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,64,0,1,float16,fp8,0,4.840784072875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,64,128,1,float16,float16,0,1.2700640360514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,64,0,1,fp8,fp8,0,2.4414666493733725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,64,0,1,float16,fp8,0,2.663402716318766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,64,128,1,float16,fp8,0,1.2523307005564372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,64,0,1,float16,float16,0,2.295461336771647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,64,128,1,fp8,fp8,0,1.635093371073405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,64,128,1,float16,float16,0,1.2879892985026042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,64,128,1,float16,fp8,0,1.297829310099284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,64,0,1,float16,fp8,0,2.396608034769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,64,0,1,fp8,fp8,0,2.14192533493042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,64,128,1,fp8,fp8,0,1.6815786361694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,64,0,1,float16,float16,0,2.3523146311442056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,64,128,1,float16,float16,0,1.298693339029948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,64,0,1,fp8,fp8,0,2.1709973017374673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,64,128,1,float16,fp8,0,1.3156479994455974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,64,0,1,float16,fp8,0,2.5218027432759604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,64,128,1,fp8,fp8,0,1.702799956003825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,64,0,1,float16,float16,0,2.371509393056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,64,128,1,float16,float16,0,1.3214720090230305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,64,0,1,float16,fp8,0,2.3815627098083496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,64,128,1,float16,fp8,0,1.3153599898020427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,64,0,1,fp8,fp8,0,2.247338612874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,64,128,1,float16,float16,0,0.7635573546091715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,64,128,1,fp8,fp8,0,1.7758879661560059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,64,0,1,float16,float16,0,2.3794506390889487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,64,128,1,float16,fp8,0,0.751477320988973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,64,128,1,fp8,fp8,0,0.9662026564280192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,64,0,1,float16,float16,0,1.3172213236490886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,64,0,1,float16,fp8,0,2.365525404612223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,64,0,1,fp8,fp8,0,2.229877312978109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,64,128,1,float16,float16,0,0.696773370107015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,64,0,1,float16,fp8,0,1.2729919751485188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,64,0,1,fp8,fp8,0,1.198463996251424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,64,128,1,float16,fp8,0,0.6599893172581991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,64,128,1,fp8,fp8,0,0.8481280008951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,64,0,1,float16,float16,0,1.1919679641723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,64,128,1,float16,float16,0,0.6850132942199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,64,0,1,float16,fp8,0,1.2030399640401204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,64,0,1,fp8,fp8,0,1.0947840213775635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,64,128,1,float16,fp8,0,0.6526133219401041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,64,0,1,float16,float16,0,1.1844000021616619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,64,128,1,fp8,fp8,0,0.8553120295206705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,64,128,1,float16,float16,0,0.6782986323038737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,64,0,1,float16,fp8,0,1.1831573645273845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,64,128,1,float16,fp8,0,0.6543733278910319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,64,0,1,fp8,fp8,0,1.1030080318450928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,64,128,1,fp8,fp8,0,0.8566880226135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,64,0,1,float16,float16,0,1.220688025156657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,64,128,1,float16,float16,0,0.6759893099466959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,64,0,1,float16,fp8,0,1.2006080150604248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,64,0,1,fp8,fp8,0,1.1077706813812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,64,128,1,float16,fp8,0,0.6579146782557169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,64,128,1,fp8,fp8,0,0.8708533445994059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,64,128,1,float16,float16,0,0.4014879862467448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,64,0,1,float16,float16,0,1.21670397122701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,64,0,1,float16,fp8,0,1.1949066321055095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,64,128,1,float16,fp8,0,0.39476267496744794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,64,0,1,fp8,fp8,0,1.138602654139201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,64,0,1,float16,float16,0,0.6803733507792155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,64,128,1,fp8,fp8,0,0.4976693391799927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,64,128,1,float16,float16,0,0.3572746515274048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,64,0,1,fp8,fp8,0,0.6268320083618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,64,0,1,float16,fp8,0,0.6644479831059774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,64,128,1,float16,fp8,0,0.3478986819585164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,64,0,1,float16,float16,0,0.6242773135503134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,64,128,1,fp8,fp8,0,0.46533334255218506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,64,128,1,float16,float16,0,0.3578559954961141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,64,0,1,float16,fp8,0,0.6146346728006998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,64,0,1,fp8,fp8,0,0.5866400003433228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,64,0,1,float16,float16,0,0.6274880170822144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,64,128,1,float16,fp8,0,0.34958934783935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,64,128,1,fp8,fp8,0,0.45844801266988117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,64,0,1,float16,fp8,0,0.6227680047353109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,64,128,1,float16,float16,0,0.3622080087661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,64,0,1,fp8,fp8,0,0.5938080151875814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,64,128,1,float16,fp8,0,0.3564266761144002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,64,128,1,fp8,fp8,0,0.4633066654205322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,64,0,1,float16,float16,0,0.6282133261362711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,64,0,1,float16,fp8,0,0.623578667640686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,64,128,1,float16,float16,0,0.36522666613260907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,64,0,1,fp8,fp8,0,0.5850773255030314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,64,128,1,float16,fp8,0,0.35659201939900714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,64,0,1,float16,float16,0,0.6444053252538046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,64,128,1,fp8,fp8,0,0.46693865458170575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,64,128,1,float16,float16,0,0.22291733821233115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,64,0,1,float16,fp8,0,0.6362026532491049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,64,0,1,fp8,fp8,0,0.5862346490224203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,64,128,1,float16,fp8,0,0.22064000368118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,64,0,1,float16,float16,0,0.35915199915568036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,64,128,1,fp8,fp8,0,0.28918933868408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,64,0,1,float16,fp8,0,0.3601386547088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,64,0,1,fp8,fp8,0,0.3324693242708842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,64,128,1,float16,float16,0,0.2039253314336141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,64,0,1,float16,float16,0,0.3413279851277669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,64,128,1,float16,fp8,0,0.19873599211374918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,64,128,1,fp8,fp8,0,0.2616106669108073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,64,0,1,float16,fp8,0,0.34069331487019855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,64,0,1,fp8,fp8,0,0.3053013285001119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,64,128,1,float16,float16,0,0.2025760014851888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,64,128,1,float16,fp8,0,0.19937600692113241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,64,0,1,fp8,fp8,0,0.30803734064102173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,64,0,1,float16,float16,0,0.34700266520182294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,64,128,1,fp8,fp8,0,0.26131733258565265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,64,0,1,float16,fp8,0,0.3412586847941081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,64,128,1,float16,float16,0,0.202239990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,64,0,1,float16,float16,0,0.3449973265329997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,64,128,1,float16,fp8,0,0.20035199324289957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,64,128,1,fp8,fp8,0,0.264575997988383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,64,0,1,float16,fp8,0,0.3421279986699422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,64,0,1,fp8,fp8,0,0.3075626691182454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,64,128,1,float16,float16,0,0.20483734210332236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,64,0,1,float16,float16,0,0.34628268082936603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,64,128,1,float16,fp8,0,0.20436267058054605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,64,128,1,float16,float16,0,0.15050133069356283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,64,128,1,fp8,fp8,0,0.2631946603457133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,64,0,1,float16,fp8,0,0.34564268589019775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,64,0,1,fp8,fp8,0,0.3104479908943176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,64,0,1,float16,float16,0,0.21127466360727945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,64,128,1,float16,fp8,0,0.15123732884724936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,64,128,1,fp8,fp8,0,0.18561599651972452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,64,0,1,float16,fp8,0,0.2121973236401876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,64,0,1,fp8,fp8,0,0.19878933827082315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,64,128,1,float16,float16,0,0.14778666694959006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,64,0,1,float16,float16,0,0.20896534125010172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,64,128,1,float16,fp8,0,0.147706667582194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,64,128,1,fp8,fp8,0,0.18179200092951456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,64,0,1,float16,fp8,0,0.20948266983032227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,64,0,1,fp8,fp8,0,0.19635732968648276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,64,128,1,float16,float16,0,0.1471733351548513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,64,0,1,float16,float16,0,0.209279994169871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,64,128,1,float16,fp8,0,0.14828266700108847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,64,128,1,fp8,fp8,0,0.17700799306233725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,64,0,1,float16,fp8,0,0.20823999245961508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,64,0,1,fp8,fp8,0,0.19754666090011597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,64,128,1,float16,float16,0,0.14823466539382935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,64,0,1,float16,float16,0,0.20859734217325845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,64,128,1,float16,fp8,0,0.14893333117167154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,64,128,1,fp8,fp8,0,0.18342934052149454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,64,0,1,float16,fp8,0,0.2089759906133016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,64,0,1,fp8,fp8,0,0.1971199909845988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,64,128,1,float16,float16,0,0.14920000235239664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,64,0,1,float16,float16,0,0.20887466271718344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,64,128,1,float16,fp8,0,0.15051199992497763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,64,128,1,fp8,fp8,0,0.18369066715240479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,64,0,1,float16,fp8,0,0.210207998752594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,64,0,1,fp8,fp8,0,0.19778666893641153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,64,128,1,float16,float16,0,3.4948107401529946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,64,128,1,float16,fp8,0,3.4469067255655923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,64,128,1,float16,float16,0,3.5138772328694663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,64,128,1,fp8,fp8,0,4.309376080830892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,64,0,1,fp8,fp8,0,4.509738604227702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,64,0,1,float16,float16,0,5.019386609395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,64,0,1,float16,fp8,0,4.98466142018636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,64,0,1,float16,float16,0,5.047994613647461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,64,128,1,float16,fp8,0,3.4398721059163413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,64,128,1,fp8,fp8,0,4.482624053955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,64,128,1,float16,float16,0,3.5089333852132163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,64,0,1,fp8,fp8,0,4.588368097941081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,64,0,1,float16,fp8,0,5.03274663289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,64,128,1,float16,fp8,0,3.4893226623535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,64,128,1,fp8,fp8,0,4.489599863688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,64,0,1,float16,float16,0,5.065295855204265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,64,0,1,float16,fp8,0,5.032304128011067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,64,128,1,float16,float16,0,3.572506586710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,64,128,1,float16,fp8,0,3.5291999181111655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,64,0,1,fp8,fp8,0,4.634463946024577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,64,0,1,float16,float16,0,5.14467207590739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,64,128,1,fp8,fp8,0,4.5557066599528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,64,128,1,float16,float16,0,2.031973361968994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,64,0,1,float16,fp8,0,5.081130663553874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,64,128,1,float16,fp8,0,1.9924319585164387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,64,0,1,float16,float16,0,2.885722796122233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,64,0,1,fp8,fp8,0,4.732240041097005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,64,128,1,fp8,fp8,0,2.5492800076802573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,64,128,1,float16,float16,0,1.6856320699055989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,64,0,1,fp8,fp8,0,2.6133227348327637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,64,0,1,float16,fp8,0,2.850949287414551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,64,128,1,float16,fp8,0,1.698805332183838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,64,0,1,float16,float16,0,2.462495962778727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,64,128,1,fp8,fp8,0,2.1365599632263184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,64,128,1,float16,float16,0,1.7309226989746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,64,0,1,fp8,fp8,0,2.237376054128011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,64,0,1,float16,fp8,0,2.4775306383768716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,64,128,1,float16,fp8,0,1.7438880602518718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,64,0,1,float16,float16,0,2.494762738545736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,64,128,1,fp8,fp8,0,2.2355359395345054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,64,0,1,float16,fp8,0,2.4813386599222818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,64,0,1,fp8,fp8,0,2.2599147160847983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,64,128,1,float16,float16,0,1.7470026016235352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,64,128,1,float16,fp8,0,1.7322559356689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,64,0,1,float16,float16,0,2.5137012799580893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,64,128,1,fp8,fp8,0,2.228677272796631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,64,128,1,float16,float16,0,1.767936070760091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,64,0,1,fp8,fp8,0,2.305189291636149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,64,0,1,float16,fp8,0,2.495888074239095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,64,128,1,float16,fp8,0,1.759701410929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,64,128,1,float16,float16,0,1.019978682200114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,64,0,1,float16,float16,0,2.5434133211771646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,64,128,1,fp8,fp8,0,2.2602453231811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,64,128,1,float16,fp8,0,0.9936426480611166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,64,0,1,float16,float16,0,1.4079680442810059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,64,0,1,fp8,fp8,0,2.3449546496073403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,64,0,1,float16,fp8,0,2.5716320673624673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,64,128,1,fp8,fp8,0,1.267034689585368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,64,128,1,float16,float16,0,0.846671978632609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,64,0,1,float16,fp8,0,1.3746132850646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,64,128,1,float16,fp8,0,0.834229310353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,64,0,1,fp8,fp8,0,1.3056533336639404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,64,0,1,float16,float16,0,1.2174452940622966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,64,128,1,fp8,fp8,0,1.097429354985555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,64,128,1,float16,float16,0,0.8531893094380697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,64,128,1,float16,fp8,0,0.8388853073120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,64,0,1,float16,fp8,0,1.248143990834554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,64,0,1,fp8,fp8,0,1.1246826648712158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,64,0,1,float16,float16,0,1.2322293122609456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,64,128,1,fp8,fp8,0,1.0851733684539795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,64,0,1,float16,fp8,0,1.210314671198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,64,128,1,float16,float16,0,0.8580213387807211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,64,0,1,fp8,fp8,0,1.1406186421712239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,64,128,1,float16,fp8,0,0.8508373101552328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,64,0,1,float16,float16,0,1.2444533507029216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,64,128,1,fp8,fp8,0,1.0974506537119548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,64,0,1,float16,fp8,0,1.2586719989776611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,64,128,1,float16,float16,0,0.8577760060628256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,64,0,1,fp8,fp8,0,1.1337653001149495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,64,128,1,float16,fp8,0,0.8593599796295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,64,0,1,float16,float16,0,1.2621760368347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,64,128,1,float16,float16,0,0.5238453149795532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,64,128,1,fp8,fp8,0,1.1282613277435303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,64,0,1,float16,fp8,0,1.2361973126729329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,64,128,1,float16,fp8,0,0.5002506573994955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,64,0,1,fp8,fp8,0,1.1551307042439778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,64,0,1,float16,float16,0,0.7048426469167074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,64,128,1,fp8,fp8,0,0.63482666015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,64,128,1,float16,float16,0,0.44591466585795086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,64,0,1,float16,fp8,0,0.7029973665873209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,64,0,1,fp8,fp8,0,0.6656266848246256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,64,128,1,float16,fp8,0,0.4298773209253947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,64,0,1,float16,float16,0,0.636405348777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,64,128,1,fp8,fp8,0,0.5693759918212891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,64,0,1,float16,fp8,0,0.6239786545435587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,64,0,1,fp8,fp8,0,0.5987253189086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,64,128,1,float16,float16,0,0.45109331607818604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,64,128,1,float16,fp8,0,0.4405706723531087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,64,0,1,float16,float16,0,0.6471946636835734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,64,128,1,fp8,fp8,0,0.5706506570180258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,64,0,1,float16,fp8,0,0.6307679812113444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,64,0,1,fp8,fp8,0,0.5889600118001302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,64,128,1,float16,float16,0,0.44229865074157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,64,0,1,float16,float16,0,0.6585493485132853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,64,128,1,float16,fp8,0,0.4431519905726115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,64,128,1,fp8,fp8,0,0.5810666481653849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,64,0,1,float16,fp8,0,0.6416266759236654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,64,0,1,fp8,fp8,0,0.5949386755625407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,64,128,1,float16,float16,0,0.4491573174794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,64,0,1,float16,float16,0,0.6439040104548136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,64,128,1,float16,fp8,0,0.44779733816782635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,64,128,1,fp8,fp8,0,0.5894506772359213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,64,128,1,float16,float16,0,0.2747573256492615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,64,0,1,fp8,fp8,0,0.6042773326237997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,64,0,1,float16,fp8,0,0.6447360118230184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,64,0,1,float16,float16,0,0.37291733423868817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,64,128,1,float16,fp8,0,0.2683626612027486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,64,128,1,fp8,fp8,0,0.34177064895629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,64,0,1,float16,fp8,0,0.37330134709676105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,64,0,1,fp8,fp8,0,0.3447786569595337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,64,128,1,float16,float16,0,0.24146666129430136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,64,0,1,float16,float16,0,0.34175999959309894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,64,128,1,float16,fp8,0,0.23817066351572672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,64,128,1,fp8,fp8,0,0.3149919907251994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,64,0,1,float16,fp8,0,0.33875731627146405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,64,128,1,float16,float16,0,0.24374399582544962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,64,0,1,fp8,fp8,0,0.3099840084711711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,64,0,1,float16,float16,0,0.34462932745615643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,64,128,1,float16,fp8,0,0.23997867107391357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,64,128,1,fp8,fp8,0,0.31701334317525226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,64,0,1,float16,fp8,0,0.3397546609242757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,64,0,1,fp8,fp8,0,0.3099786639213562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,64,128,1,float16,float16,0,0.24550400177637735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,64,0,1,float16,float16,0,0.3470613161722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,64,128,1,float16,fp8,0,0.24200532833735147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,64,128,1,fp8,fp8,0,0.3195306658744812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,64,0,1,float16,fp8,0,0.34302934010823566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,64,0,1,fp8,fp8,0,0.3118879993756612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,64,128,1,float16,float16,0,0.24707732597986856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,64,0,1,float16,float16,0,0.34975465138753253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,64,128,1,float16,fp8,0,0.24639999866485596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,64,128,1,fp8,fp8,0,0.32149332761764526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,64,0,1,float16,fp8,0,0.34716800848642987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,64,0,1,fp8,fp8,0,0.3164586623509725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,64,128,1,float16,float16,0,0.1535040040810903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,64,0,1,float16,float16,0,0.20195732514063516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,64,128,1,float16,fp8,0,0.15638933579126993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,64,128,1,fp8,fp8,0,0.20121600230534872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,64,0,1,float16,fp8,0,0.20238399505615234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,64,0,1,fp8,fp8,0,0.18950400749842325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,64,128,1,float16,float16,0,0.14065066973368326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,64,0,1,float16,float16,0,0.18216532468795776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,64,128,1,float16,fp8,0,0.1400159994761149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,64,128,1,fp8,fp8,0,0.1868106722831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,64,0,1,float16,fp8,0,0.1815626621246338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,64,0,1,fp8,fp8,0,0.17377066612243652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,64,0,1,float16,fp8,0,0.1818880041440328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,64,128,1,float16,float16,0,0.1406719982624054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,64,0,1,float16,float16,0,0.18338133891423544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,64,128,1,float16,fp8,0,0.14058666427930197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,64,128,1,fp8,fp8,0,0.18597867091496786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,64,0,1,fp8,fp8,0,0.17548267046610513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,64,128,1,float16,float16,0,0.1420693298180898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,64,0,1,float16,float16,0,0.1837760011355082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,64,128,1,float16,fp8,0,0.14140266180038452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,64,128,1,fp8,fp8,0,0.18652266263961792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,64,0,1,float16,fp8,0,0.1822133262952169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,64,0,1,fp8,fp8,0,0.17534933487574259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,64,0,1,fp8,fp8,0,0.1766080061594645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,64,128,1,float16,float16,0,0.1434719959894816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,64,0,1,float16,float16,0,0.18663465976715088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,64,128,1,float16,fp8,0,0.14272000392278036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,64,128,1,fp8,fp8,0,0.18821867307027182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,64,0,1,float16,fp8,0,0.18426666657129923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,64,128,1,float16,float16,0,0.10905599594116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,64,0,1,float16,float16,0,0.12778666615486145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,64,128,1,float16,fp8,0,0.10839999715487163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,64,128,1,fp8,fp8,0,0.13547733426094055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,64,0,1,float16,fp8,0,0.1274773379166921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,64,0,1,fp8,fp8,0,0.12034666538238525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,64,128,1,float16,float16,0,0.10371733705202739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,64,0,1,float16,float16,0,0.12376532951990764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,64,128,1,float16,fp8,0,0.10307199756304423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,64,128,1,fp8,fp8,0,0.12408000230789185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,64,0,1,float16,fp8,0,0.1237333317597707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,64,0,1,fp8,fp8,0,0.11768000324567159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,64,128,1,float16,float16,0,0.10345066587130229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,64,0,1,float16,float16,0,0.12381333112716675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,64,128,1,float16,fp8,0,0.10393066207567851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,64,128,1,fp8,fp8,0,0.12830400466918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,64,0,1,float16,fp8,0,0.12429333726565044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,64,0,1,fp8,fp8,0,0.11793067057927449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,64,128,1,float16,float16,0,0.10315199693044026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,64,0,1,float16,float16,0,0.12389866511027019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,64,128,1,float16,fp8,0,0.10393066207567851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,64,128,1,float16,fp8,0,0.10359467069307964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,64,128,1,fp8,fp8,0,0.1258026659488678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,64,0,1,float16,fp8,0,0.12404266993204753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,64,0,1,fp8,fp8,0,0.11843732992808025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,64,128,1,float16,float16,0,0.1051573355992635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,64,0,1,float16,float16,0,0.12390399972597758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,64,128,1,fp8,fp8,0,0.126202662785848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,64,0,1,float16,fp8,0,0.12482666969299316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,64,0,1,fp8,fp8,0,0.11839999755223592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,64,128,1,float16,float16,0,2.5922560691833496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,64,128,1,float16,fp8,0,2.552117347717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,64,0,1,float16,float16,0,3.296330769856771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,64,128,1,fp8,fp8,0,3.1954345703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,64,128,1,float16,float16,0,2.5801599820454917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,64,0,1,fp8,fp8,0,2.921120007832845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,64,0,1,float16,fp8,0,3.25930118560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,64,0,1,float16,float16,0,3.2806240717569985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,64,128,1,float16,fp8,0,2.5442026456197104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,64,128,1,fp8,fp8,0,3.2385600407918296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,64,0,1,float16,fp8,0,3.2600266138712564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,64,128,1,float16,float16,0,2.590847969055176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,64,0,1,fp8,fp8,0,2.9409494400024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,64,128,1,float16,fp8,0,2.565605322519938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,64,0,1,float16,float16,0,3.3014612197875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,64,128,1,fp8,fp8,0,3.2749547958374023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,64,0,1,float16,fp8,0,3.265791893005371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,64,128,1,float16,float16,0,2.634906609853109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,64,0,1,fp8,fp8,0,2.989450772603353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,64,128,1,float16,fp8,0,2.590720017751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,64,0,1,float16,float16,0,3.3318773905436196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,64,128,1,fp8,fp8,0,3.406522750854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,64,128,1,float16,float16,0,1.5206185976664226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,64,0,1,float16,fp8,0,3.301957448323568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,64,128,1,float16,fp8,0,1.4830613136291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,64,0,1,float16,float16,0,1.8689440091451008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,64,128,1,fp8,fp8,0,1.887781302134196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,64,0,1,float16,fp8,0,1.8587946891784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,64,0,1,fp8,fp8,0,3.1093600591023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,64,0,1,fp8,fp8,0,1.723024050394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,64,128,1,float16,float16,0,1.2492480278015137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,64,128,1,float16,fp8,0,1.2292533715565999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,64,0,1,float16,float16,0,1.5982933044433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,64,128,1,fp8,fp8,0,1.6134026845296223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,64,0,1,float16,fp8,0,1.5913546880086262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,64,128,1,float16,float16,0,1.2595893541971843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,64,0,1,fp8,fp8,0,1.481925328572591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,64,128,1,float16,fp8,0,1.2556479771931965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,64,0,1,float16,float16,0,1.6108585993448894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,64,128,1,fp8,fp8,0,1.622037410736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,64,0,1,fp8,fp8,0,1.5115413665771484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,64,0,1,float16,fp8,0,1.604746659596761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,64,128,1,float16,float16,0,1.2852853139241536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,64,128,1,float16,fp8,0,1.2679466406504314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,64,0,1,float16,float16,0,1.6220320065816243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,64,128,1,fp8,fp8,0,1.6382239659627278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,64,0,1,float16,fp8,0,1.610197385152181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,64,0,1,fp8,fp8,0,1.492794672648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,64,128,1,float16,float16,0,1.3240959644317627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,64,128,1,float16,fp8,0,1.2914186318715413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,64,0,1,float16,float16,0,1.6525813738505046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,64,128,1,float16,float16,0,0.7572693030039469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,64,128,1,fp8,fp8,0,1.6699360211690266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,64,0,1,float16,fp8,0,1.6317599614461262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,64,0,1,fp8,fp8,0,1.5196320215861003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,64,128,1,float16,fp8,0,0.7430240313212076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,64,0,1,float16,float16,0,0.9312746524810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,64,128,1,fp8,fp8,0,0.9327200253804525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,64,0,1,float16,fp8,0,0.9171413580576578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,64,128,1,float16,float16,0,0.6380533377329508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,64,0,1,fp8,fp8,0,0.8463626702626547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,64,128,1,float16,fp8,0,0.6271093289057413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,64,128,1,fp8,fp8,0,0.8260853290557861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,64,0,1,float16,float16,0,0.812005360921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,64,0,1,float16,fp8,0,0.7946986357371012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,64,0,1,fp8,fp8,0,0.744757334391276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,64,128,1,float16,float16,0,0.6367413202921549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,64,0,1,float16,float16,0,0.8201013406117758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,64,128,1,float16,fp8,0,0.6264479955037435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,64,128,1,fp8,fp8,0,0.8238453070322672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,64,0,1,float16,fp8,0,0.8155573209126791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,64,128,1,float16,float16,0,0.6358346541722616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,64,0,1,fp8,fp8,0,0.7453013261159261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,64,0,1,float16,float16,0,0.8243359724680582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,64,128,1,float16,fp8,0,0.6322826544443766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,64,128,1,fp8,fp8,0,0.8311200141906738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,64,0,1,float16,fp8,0,0.8081493377685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,64,0,1,fp8,fp8,0,0.7508906523386637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,64,128,1,float16,float16,0,0.6521226565043131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,64,128,1,float16,fp8,0,0.6414026816685995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,64,0,1,float16,float16,0,0.8193493684132894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,64,128,1,fp8,fp8,0,0.8425760269165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,64,128,1,float16,float16,0,0.3904586633046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,64,0,1,float16,fp8,0,0.8202773729960123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,64,0,1,float16,float16,0,0.48110934098561603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,64,0,1,fp8,fp8,0,0.7634826501210531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,64,128,1,float16,fp8,0,0.3835039933522542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,64,128,1,fp8,fp8,0,0.480405330657959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,64,0,1,float16,fp8,0,0.4695306619008382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,64,0,1,fp8,fp8,0,0.43836267789204914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,64,128,1,float16,float16,0,0.33503464857737225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,64,0,1,float16,float16,0,0.4291573365529378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,64,128,1,float16,fp8,0,0.3302239974339803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,64,128,1,fp8,fp8,0,0.4363360007603963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,64,0,1,float16,fp8,0,0.4240320126215617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,64,0,1,fp8,fp8,0,0.3922400077184041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,64,128,1,float16,float16,0,0.3418613274892171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,64,0,1,float16,float16,0,0.42976534366607666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,64,128,1,float16,fp8,0,0.33186133702596027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,64,128,1,fp8,fp8,0,0.4394559860229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,64,0,1,float16,fp8,0,0.4232213497161865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,64,0,1,fp8,fp8,0,0.3948320150375366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,64,128,1,float16,float16,0,0.3413279851277669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,64,0,1,float16,float16,0,0.43188798427581787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,64,128,1,float16,float16,0,0.34985601902008057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,64,128,1,float16,fp8,0,0.3402080138524373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,64,128,1,fp8,fp8,0,0.4416746695836385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,64,0,1,float16,fp8,0,0.4267520109812419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,64,0,1,fp8,fp8,0,0.3970186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,64,0,1,float16,float16,0,0.4354506731033325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,64,128,1,float16,fp8,0,0.34037331740061444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,64,128,1,fp8,fp8,0,0.44576533635457355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,64,0,1,float16,fp8,0,0.4389066696166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,64,128,1,float16,float16,0,0.2095680038134257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,64,0,1,float16,fp8,0,0.2548853357632955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,64,0,1,fp8,fp8,0,0.40229864915211994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,64,0,1,float16,float16,0,0.2577333251635234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,64,128,1,float16,fp8,0,0.20702399810155234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,64,128,1,fp8,fp8,0,0.26768000920613605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,64,0,1,fp8,fp8,0,0.23820799589157104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,64,128,1,float16,float16,0,0.18427733580271402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,64,128,1,fp8,fp8,0,0.24210667610168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,64,0,1,float16,float16,0,0.23259733120600382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,64,128,1,float16,fp8,0,0.18288000424702963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,64,0,1,float16,fp8,0,0.2303253412246704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,64,0,1,fp8,fp8,0,0.20960533618927002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,64,128,1,float16,float16,0,0.18850666284561157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,64,128,1,float16,fp8,0,0.18316799402236938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,64,0,1,float16,float16,0,0.2339199980099996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,64,128,1,fp8,fp8,0,0.24346667528152466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,64,0,1,float16,fp8,0,0.2323039968808492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,64,0,1,fp8,fp8,0,0.21043733755747476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,64,128,1,float16,float16,0,0.18820800383885702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,64,0,1,float16,float16,0,0.23725332816441855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,64,128,1,float16,fp8,0,0.18460800250371298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,64,128,1,fp8,fp8,0,0.2455893357594808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,64,0,1,float16,fp8,0,0.23310399055480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,64,0,1,fp8,fp8,0,0.21177599827448526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,64,128,1,float16,float16,0,0.19031999508539835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,64,0,1,float16,float16,0,0.23830399910608926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,64,0,1,fp8,fp8,0,0.21485867102940878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,64,128,1,float16,fp8,0,0.1909546653429667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,64,128,1,fp8,fp8,0,0.2476053237915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,64,0,1,float16,fp8,0,0.23497066895167032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,64,128,1,float16,float16,0,0.12148800492286682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,64,0,1,float16,float16,0,0.13913066188494363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,64,128,1,float16,fp8,0,0.12325867017110188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,64,128,1,fp8,fp8,0,0.15915733575820923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,64,0,1,float16,fp8,0,0.141893337170283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,64,0,1,fp8,fp8,0,0.13376532991727194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,64,128,1,float16,float16,0,0.10943466424942017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,64,0,1,float16,float16,0,0.1262453297773997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,64,128,1,float16,fp8,0,0.11042666435241699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,64,128,1,float16,fp8,0,0.10956799983978271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,64,128,1,fp8,fp8,0,0.1432213286558787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,64,0,1,float16,fp8,0,0.12641599774360657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,64,0,1,fp8,fp8,0,0.11999467015266418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,64,0,1,float16,float16,0,0.12661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,64,128,1,float16,float16,0,0.11061867078145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,64,128,1,fp8,fp8,0,0.1455413301785787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,64,0,1,float16,fp8,0,0.12591999769210815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,64,0,1,fp8,fp8,0,0.12025066216786702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,64,128,1,float16,float16,0,0.11074666182200114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,64,0,1,float16,float16,0,0.1274773379166921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,64,128,1,float16,fp8,0,0.11082667112350464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,64,128,1,fp8,fp8,0,0.1458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,64,128,1,float16,fp8,0,0.11199466387430827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,64,0,1,float16,fp8,0,0.12702932953834534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,64,0,1,fp8,fp8,0,0.12199466427167256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,64,128,1,float16,float16,0,0.1125866671403249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,64,0,1,float16,float16,0,0.12814933061599731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,64,128,1,fp8,fp8,0,0.14941866199175516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,64,0,1,float16,fp8,0,0.12877866625785828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,64,128,1,float16,float16,0,0.08570667107899983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,64,0,1,fp8,fp8,0,0.12485333283742268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,64,0,1,float16,float16,0,0.09274666508038838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,64,128,1,float16,fp8,0,0.08644800384839375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,64,128,1,fp8,fp8,0,0.10950400431950887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,64,0,1,float16,fp8,0,0.09326400359471639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,64,0,1,fp8,fp8,0,0.08930666248003642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,64,128,1,float16,float16,0,0.08158400158087413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,64,0,1,float16,float16,0,0.090037335952123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,64,128,1,float16,fp8,0,0.0819413314263026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,64,128,1,fp8,fp8,0,0.09877333045005798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,64,0,1,float16,fp8,0,0.0904906690120697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,64,0,1,fp8,fp8,0,0.0867039958635966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,64,0,1,fp8,fp8,0,0.0865066647529602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,64,128,1,float16,float16,0,0.08207466701666515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,64,0,1,float16,float16,0,0.09030399719874065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,64,128,1,float16,fp8,0,0.08216000099976857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,64,128,1,fp8,fp8,0,0.0981760025024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,64,0,1,float16,fp8,0,0.09035733342170715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,64,128,1,float16,float16,0,0.0817493349313736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,64,0,1,float16,float16,0,0.09033067027727763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,64,128,1,float16,fp8,0,0.0819946676492691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,64,128,1,fp8,fp8,0,0.09876799583435059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,64,0,1,float16,fp8,0,0.08994666735331218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,64,0,1,float16,fp8,0,0.09027199943860371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,64,0,1,fp8,fp8,0,0.08704533179601033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,64,128,1,float16,float16,0,0.08230933547019958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,64,0,1,float16,float16,0,0.09031466643015544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,64,128,1,float16,fp8,0,0.08188266555468242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,64,128,1,fp8,fp8,0,0.10025599598884583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,64,0,1,fp8,fp8,0,0.08634666601816814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,64,128,1,float16,float16,0,2.9861119588216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,64,128,1,float16,fp8,0,3.002613385518392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,64,0,1,float16,float16,0,3.4230400721232095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,64,0,1,fp8,fp8,0,3.0674559275309243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,64,128,1,float16,float16,0,3.0040693283081055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,64,0,1,float16,fp8,0,3.429258664449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,64,128,1,fp8,fp8,0,3.937258720397949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,64,0,1,float16,float16,0,3.450842539469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,64,128,1,float16,fp8,0,3.0162881215413413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,64,0,1,fp8,fp8,0,3.093455950419108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,64,128,1,float16,float16,0,3.0246025721232095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,64,0,1,float16,fp8,0,3.427562713623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,64,128,1,float16,fp8,0,3.0338560740152993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,64,128,1,fp8,fp8,0,3.971861203511556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,64,0,1,float16,float16,0,3.462623914082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,64,128,1,fp8,fp8,0,3.9418932596842446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,64,0,1,float16,fp8,0,3.438490549723307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,64,0,1,fp8,fp8,0,3.0943625768025718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,64,128,1,float16,float16,0,3.225872039794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,64,128,1,float16,fp8,0,3.1939306259155273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,64,0,1,float16,float16,0,3.65721066792806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,64,128,1,fp8,fp8,0,4.01694933573405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,64,128,1,float16,float16,0,1.894938627878825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,64,0,1,float16,float16,0,2.118394692738851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,64,0,1,float16,fp8,0,3.612629254659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,64,128,1,float16,fp8,0,1.8181227048238118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,64,0,1,fp8,fp8,0,3.1313279469807944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,64,0,1,float16,fp8,0,2.0423146883646646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,64,128,1,fp8,fp8,0,2.250821272532145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,64,128,1,float16,float16,0,1.4736587206522624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,64,0,1,fp8,fp8,0,1.8008480072021484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,64,0,1,float16,float16,0,1.7350400288899739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,64,128,1,float16,fp8,0,1.471567948659261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,64,0,1,float16,fp8,0,1.685167948404948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,64,0,1,fp8,fp8,0,1.49944003423055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,64,128,1,fp8,fp8,0,1.931104024251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,64,128,1,float16,float16,0,1.5126880009969075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,64,0,1,float16,float16,0,1.7100213368733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,64,128,1,float16,fp8,0,1.5223466555277507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,64,0,1,float16,fp8,0,1.7050293286641438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,64,0,1,fp8,fp8,0,1.5141706466674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,64,128,1,fp8,fp8,0,2.0115626653035483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,64,128,1,float16,float16,0,1.5103786786397297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,64,128,1,float16,fp8,0,1.5253440539042156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,64,0,1,float16,float16,0,1.7243520418802898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,64,0,1,float16,fp8,0,1.765077273050944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,64,128,1,fp8,fp8,0,1.9660639762878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,64,0,1,fp8,fp8,0,1.5276533762613933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,64,128,1,float16,float16,0,1.6191840171813965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,64,0,1,float16,float16,0,1.8180853525797527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,64,128,1,float16,fp8,0,1.5763519605000813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,64,128,1,float16,float16,0,0.9436639944712321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,64,0,1,float16,fp8,0,1.797573407491048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,64,128,1,fp8,fp8,0,2.0222880045572915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,64,0,1,float16,float16,0,1.0509173075358074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,64,128,1,float16,fp8,0,0.9289813041687012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,64,0,1,fp8,fp8,0,1.5832160313924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,64,128,1,fp8,fp8,0,1.1232373714447021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,64,0,1,float16,fp8,0,1.0212480227152507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,64,0,1,fp8,fp8,0,0.8911946614583334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,64,128,1,float16,float16,0,0.7502293586730957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,64,0,1,float16,float16,0,0.8838826815287272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,64,128,1,float16,fp8,0,0.7505119641621908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,64,0,1,float16,fp8,0,0.8731253147125244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,64,128,1,fp8,fp8,0,1.0072320302327473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,64,0,1,fp8,fp8,0,0.7829919656117758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,64,128,1,float16,float16,0,0.7599786917368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,64,128,1,float16,fp8,0,0.76091202100118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,64,0,1,float16,float16,0,0.8939146995544434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,64,128,1,fp8,fp8,0,1.0141973495483398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,64,0,1,float16,fp8,0,0.8671680291493734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,64,0,1,fp8,fp8,0,0.7718559900919596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,64,128,1,float16,float16,0,0.7794453303019205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,64,0,1,float16,float16,0,0.8958986600240072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,64,128,1,float16,fp8,0,0.7675999800364176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,64,0,1,float16,fp8,0,0.9000426928202311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,64,128,1,fp8,fp8,0,0.9913919766743978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,64,0,1,fp8,fp8,0,0.7893813451131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,64,128,1,float16,float16,0,0.7883040110270182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,64,0,1,float16,float16,0,0.9022133350372314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,64,128,1,float16,fp8,0,0.7912267049153646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,64,128,1,float16,float16,0,0.48130667209625244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,64,0,1,fp8,fp8,0,0.8094879786173502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,64,128,1,fp8,fp8,0,1.0264960130055745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,64,0,1,float16,fp8,0,0.893893321355184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,64,0,1,float16,float16,0,0.5452213287353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,64,128,1,float16,fp8,0,0.4698293209075928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,64,128,1,fp8,fp8,0,0.5734613339106241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,64,0,1,fp8,fp8,0,0.44949865341186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,64,0,1,float16,fp8,0,0.5272906621297201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,64,128,1,float16,float16,0,0.3957066535949707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,64,0,1,float16,float16,0,0.45235733191172284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,64,128,1,float16,fp8,0,0.3910133441289266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,64,128,1,fp8,fp8,0,0.5285600026448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,64,0,1,float16,fp8,0,0.4455146789550781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,64,0,1,fp8,fp8,0,0.39867734909057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,64,128,1,float16,float16,0,0.39477332433064777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,64,0,1,float16,float16,0,0.45296533902486164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,64,128,1,float16,fp8,0,0.3996640046437581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,64,128,1,fp8,fp8,0,0.5222560167312622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,64,0,1,float16,fp8,0,0.44947731494903564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,64,0,1,fp8,fp8,0,0.40464532375335693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,64,128,1,float16,float16,0,0.39823468526204425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,64,0,1,float16,float16,0,0.45459731419881183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,64,128,1,float16,fp8,0,0.3989866574605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,64,128,1,fp8,fp8,0,0.5226080020268759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,64,0,1,float16,fp8,0,0.45865599314371747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,64,0,1,fp8,fp8,0,0.41094934940338135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,64,128,1,float16,float16,0,0.4063946803410848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,64,0,1,float16,float16,0,0.46278401215871173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,64,128,1,float16,fp8,0,0.4073599974314372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,64,128,1,fp8,fp8,0,0.5218026638031006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,64,0,1,float16,fp8,0,0.46326935291290283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,64,128,1,float16,float16,0,0.2575146754582723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,64,0,1,fp8,fp8,0,0.4095146656036377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,64,0,1,float16,float16,0,0.28117332855860394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,64,128,1,float16,fp8,0,0.24832000335057577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,64,128,1,fp8,fp8,0,0.3048373262087504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,64,0,1,float16,fp8,0,0.2786293427149455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,64,0,1,fp8,fp8,0,0.23746132850646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,64,128,1,float16,float16,0,0.21550933519999185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,64,0,1,float16,float16,0,0.24028799931208292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,64,128,1,float16,fp8,0,0.21202667554219565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,64,128,1,fp8,fp8,0,0.27877867221832275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,64,0,1,float16,fp8,0,0.24013866980870566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,64,0,1,fp8,fp8,0,0.2116640011469523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,64,128,1,float16,float16,0,0.21363200743993124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,64,0,1,float16,float16,0,0.24382932980855307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,64,128,1,float16,fp8,0,0.2156320015589396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,64,128,1,fp8,fp8,0,0.28042133649190265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,64,0,1,float16,fp8,0,0.24261866013209024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,64,0,1,fp8,fp8,0,0.20959466695785522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,64,128,1,float16,float16,0,0.21762667099634805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,64,0,1,float16,float16,0,0.24290132522583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,64,128,1,float16,fp8,0,0.21874666213989258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,64,128,1,fp8,fp8,0,0.2831679979960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,64,0,1,float16,fp8,0,0.24347732464472452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,64,0,1,fp8,fp8,0,0.21144533157348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,64,128,1,float16,float16,0,0.21698667605717978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,64,0,1,float16,float16,0,0.2472213308016459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,64,128,1,float16,fp8,0,0.2175146738688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,64,128,1,fp8,fp8,0,0.2847519914309184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,64,0,1,float16,fp8,0,0.24548800786336264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,64,0,1,fp8,fp8,0,0.21388266483942667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,64,128,1,float16,float16,0,0.13849600156148276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,64,0,1,float16,float16,0,0.1566933294137319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,64,128,1,float16,float16,0,0.11851200461387634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,64,128,1,float16,fp8,0,0.1376586655775706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,64,128,1,fp8,fp8,0,0.1718719998995463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,64,0,1,float16,fp8,0,0.15239999691645303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,64,0,1,fp8,fp8,0,0.12957866986592612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,64,0,1,float16,float16,0,0.12707733114560446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,64,128,1,float16,fp8,0,0.1185706655184428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,64,128,1,fp8,fp8,0,0.15737600127855936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,64,128,1,float16,fp8,0,0.11938132842381795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,64,128,1,fp8,fp8,0,0.1583626667658488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,64,0,1,float16,float16,0,0.12811199824015299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,64,0,1,float16,fp8,0,0.12820800145467123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,64,0,1,fp8,fp8,0,0.11471999684969585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,64,128,1,float16,float16,0,0.11876799662907918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,64,128,1,fp8,fp8,0,0.16105600198109946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,64,0,1,float16,fp8,0,0.12811199824015299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,64,0,1,fp8,fp8,0,0.11659733454386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,64,128,1,float16,float16,0,0.12017066280047099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,64,0,1,float16,float16,0,0.13012799620628357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,64,128,1,float16,fp8,0,0.12095466256141663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,64,0,1,float16,fp8,0,0.12967466314633688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,64,0,1,fp8,fp8,0,0.11737599968910217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,64,128,1,float16,float16,0,0.12139200170834859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,64,0,1,float16,float16,0,0.12966400384902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,64,128,1,float16,fp8,0,0.12282133102416992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,64,128,1,fp8,fp8,0,0.1599573294321696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,64,0,1,float16,fp8,0,0.13149333000183105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,64,0,1,fp8,fp8,0,0.11844266454378764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,64,128,1,float16,float16,0,0.07935466865698497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,64,0,1,float16,float16,0,0.08196266492207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,64,128,1,float16,fp8,0,0.07974400122960408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,64,128,1,fp8,fp8,0,0.10357866684595744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,64,0,1,float16,fp8,0,0.08090133468310039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,64,0,1,fp8,fp8,0,0.07381866872310638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,64,128,1,float16,float16,0,0.06916800141334534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,64,0,1,float16,float16,0,0.07141333321730296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,64,0,1,float16,float16,0,0.0717386653025945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,64,128,1,float16,fp8,0,0.07001600166161855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,64,128,1,fp8,fp8,0,0.08888000249862671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,64,0,1,float16,fp8,0,0.07240533332029979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,64,0,1,fp8,fp8,0,0.06472533444563548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,64,128,1,float16,float16,0,0.06898666421572368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,64,128,1,float16,fp8,0,0.0695360004901886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,64,128,1,fp8,fp8,0,0.08850133419036865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,64,0,1,float16,fp8,0,0.07180266578992207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,64,0,1,fp8,fp8,0,0.06555200119813283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,64,128,1,float16,float16,0,0.07176533341407776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,64,0,1,float16,float16,0,0.07334933181603749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,64,128,1,float16,fp8,0,0.07039466500282288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,64,128,1,fp8,fp8,0,0.08905599514643352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,64,0,1,float16,fp8,0,0.07306666672229767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,64,0,1,fp8,fp8,0,0.06651199857393901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,64,128,1,float16,float16,0,0.07083199918270111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,64,0,1,float16,float16,0,0.07458133498827617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,64,128,1,float16,fp8,0,0.07187733550866444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,64,128,1,fp8,fp8,0,0.09217600027720134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,64,0,1,float16,fp8,0,0.07402133444945018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,64,0,1,fp8,fp8,0,0.06709866722424825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,64,128,1,float16,float16,0,0.05020800232887268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,64,0,1,float16,float16,0,0.05014933149019877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,64,128,1,float16,fp8,0,0.04933866858482361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,64,128,1,fp8,fp8,0,0.05909866591294607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,64,0,1,float16,fp8,0,0.0495306650797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,64,0,1,fp8,fp8,0,0.04595200220743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,64,128,1,float16,float16,0,0.04669333497683207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,64,0,1,float16,float16,0,0.04752533137798309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,64,128,1,float16,fp8,0,0.04699199895064036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,64,128,1,fp8,fp8,0,0.057189335425694786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,64,0,1,float16,fp8,0,0.04609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,64,0,1,fp8,fp8,0,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,64,0,1,float16,fp8,0,0.04739200075467428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,64,128,1,float16,float16,0,0.047509332497914634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,64,0,1,float16,float16,0,0.04757333298524221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,64,128,1,float16,fp8,0,0.04642133414745331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,64,128,1,fp8,fp8,0,0.05629333357016245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,64,0,1,fp8,fp8,0,0.043231998880704246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,64,128,1,float16,float16,0,0.046629334489504494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,64,0,1,float16,float16,0,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,64,128,1,float16,fp8,0,0.04756799836953481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,64,128,1,fp8,fp8,0,0.05725333094596863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,64,0,1,float16,fp8,0,0.04753066599369049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,64,0,1,fp8,fp8,0,0.04420800010363261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,64,0,1,float16,fp8,0,0.04814399778842926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,64,128,1,float16,float16,0,0.04717866579691569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,64,0,1,float16,float16,0,0.048058668772379555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,64,128,1,float16,fp8,0,0.047557334105173744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,64,128,1,fp8,fp8,0,0.05775466561317444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,64,0,1,fp8,fp8,0,0.044768000642458596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,64,128,1,float16,float16,0,2.816181182861328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,64,0,1,float16,float16,0,2.8009866078694663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,64,128,1,float16,fp8,0,2.8415145874023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,64,0,1,float16,fp8,0,2.800645192464193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,64,0,1,fp8,fp8,0,2.4656426111857095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,64,128,1,float16,float16,0,2.874143918355306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,64,128,1,fp8,fp8,0,3.7141599655151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,64,0,1,float16,float16,0,2.821338653564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,64,128,1,float16,fp8,0,2.8569065729777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,64,0,1,fp8,fp8,0,2.5086453755696616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,64,0,1,float16,fp8,0,2.8222986857096353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,64,128,1,fp8,fp8,0,3.7658507029215493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,64,128,1,float16,float16,0,2.8692267735799155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,64,0,1,float16,float16,0,2.844623883565267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,64,128,1,float16,fp8,0,2.887322743733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,64,0,1,fp8,fp8,0,2.515925407409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,64,0,1,float16,fp8,0,2.831226666768392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,64,128,1,float16,float16,0,3.0503412882486978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,64,128,1,fp8,fp8,0,3.707765261332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,64,0,1,float16,float16,0,3.049824078877767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,64,128,1,float16,fp8,0,3.034261385599772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,64,128,1,fp8,fp8,0,3.782794634501139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,64,0,1,float16,fp8,0,3.0336745580037436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,64,128,1,float16,float16,0,1.8199572563171387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,64,0,1,float16,float16,0,1.7972960472106934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,64,128,1,float16,fp8,0,1.7496906916300456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,64,0,1,fp8,fp8,0,2.5760107040405273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,64,0,1,float16,fp8,0,1.7348960240681965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,64,128,1,fp8,fp8,0,2.1258719762166343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,64,0,1,fp8,fp8,0,1.4953440030415852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,64,128,1,float16,float16,0,1.4126133918762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,64,0,1,float16,float16,0,1.386624018351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,64,128,1,float16,fp8,0,1.4326772689819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,64,0,1,float16,fp8,0,1.3940745989481609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,64,128,1,fp8,fp8,0,1.8242133458455403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,64,0,1,fp8,fp8,0,1.247056007385254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,64,128,1,float16,float16,0,1.4298559824625652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,64,0,1,float16,float16,0,1.396506627400716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,64,128,1,float16,fp8,0,1.4325493176778157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,64,128,1,fp8,fp8,0,1.8523732821146648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,64,0,1,fp8,fp8,0,1.2300480206807454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,64,0,1,float16,fp8,0,1.41759459177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,64,128,1,float16,float16,0,1.4214773178100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,64,0,1,float16,float16,0,1.4483253161112468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,64,128,1,float16,fp8,0,1.4356853167215984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,64,0,1,float16,fp8,0,1.4193226496378581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,64,128,1,fp8,fp8,0,1.8496692975362141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,64,0,1,fp8,fp8,0,1.2378613154093425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,64,128,1,float16,float16,0,1.5188639958699544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,64,0,1,float16,float16,0,1.5185386339823406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,64,128,1,float16,fp8,0,1.5039626757303874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,64,128,1,fp8,fp8,0,1.9105653762817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,64,128,1,float16,float16,0,0.908191998799642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,64,0,1,float16,float16,0,0.9060640335083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,64,0,1,float16,fp8,0,1.500970681508382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,64,0,1,fp8,fp8,0,1.2740106582641602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,64,128,1,float16,fp8,0,0.8805226484934489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,64,0,1,float16,fp8,0,0.8594773610432943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,64,128,1,fp8,fp8,0,1.069925308227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,64,0,1,fp8,fp8,0,0.7524212996164957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,64,128,1,float16,float16,0,0.7122986316680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,64,0,1,float16,float16,0,0.7220479647318522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,64,128,1,float16,fp8,0,0.7163093090057373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,64,0,1,float16,fp8,0,0.7051359812418619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,64,128,1,fp8,fp8,0,0.932586669921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,64,0,1,fp8,fp8,0,0.6083093484242758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,64,128,1,float16,float16,0,0.737663984298706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,64,0,1,float16,float16,0,0.7308693726857504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,64,128,1,float16,fp8,0,0.7232159773508707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,64,0,1,float16,fp8,0,0.7155360380808512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,64,128,1,fp8,fp8,0,0.9553279876708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,64,0,1,fp8,fp8,0,0.6177599827448527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,64,128,1,float16,float16,0,0.7361120382944742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,64,0,1,float16,float16,0,0.7162133057912191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,64,128,1,float16,fp8,0,0.7438293298085531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,64,0,1,float16,fp8,0,0.7229333718617758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,64,0,1,fp8,fp8,0,0.6351840098698934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,64,128,1,fp8,fp8,0,0.9353546301523844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,64,128,1,float16,float16,0,0.7507200241088867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,64,0,1,float16,float16,0,0.7461280028025309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,64,128,1,float16,fp8,0,0.7459413210550944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,64,0,1,float16,fp8,0,0.737285296122233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,64,128,1,fp8,fp8,0,0.96670929590861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,64,128,1,float16,float16,0,0.46237866083780926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,64,0,1,fp8,fp8,0,0.653493324915568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,64,0,1,float16,float16,0,0.45468799273173016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,64,128,1,float16,fp8,0,0.4526880184809367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,64,128,1,fp8,fp8,0,0.541264017422994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,64,0,1,float16,fp8,0,0.44177067279815674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,64,0,1,fp8,fp8,0,0.3766239881515503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,64,128,1,float16,float16,0,0.37057065963745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,64,0,1,float16,float16,0,0.3694506486256917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,64,128,1,float16,fp8,0,0.37466665108998615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,64,128,1,fp8,fp8,0,0.4859040180842082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,64,0,1,float16,fp8,0,0.36397866408030194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,64,0,1,fp8,fp8,0,0.3224800030390422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,64,128,1,float16,float16,0,0.3790080149968465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,64,128,1,fp8,fp8,0,0.48872001965840656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,64,0,1,float16,float16,0,0.3686986764272054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,64,128,1,float16,fp8,0,0.38095998764038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,64,0,1,float16,fp8,0,0.3686559995015462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,64,0,1,fp8,fp8,0,0.3242400089899699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,64,128,1,float16,float16,0,0.38098665078481037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,64,0,1,float16,float16,0,0.3738186756769816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,64,128,1,float16,fp8,0,0.38066665331522626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,64,128,1,fp8,fp8,0,0.49185065428415936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,64,0,1,float16,fp8,0,0.37622400124867755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,64,0,1,fp8,fp8,0,0.32652799288431805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,64,128,1,float16,float16,0,0.38946131865183514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,64,0,1,float16,float16,0,0.3808639844258626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,64,128,1,float16,fp8,0,0.38763733704884845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,64,128,1,fp8,fp8,0,0.49513065814971924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,64,0,1,float16,fp8,0,0.37881600856781006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,64,0,1,fp8,fp8,0,0.33275200923283893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,64,128,1,float16,float16,0,0.245253324508667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,64,0,1,float16,float16,0,0.2384213407834371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,64,128,1,float16,fp8,0,0.23806933561960855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,64,128,1,fp8,fp8,0,0.2895466685295105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,64,0,1,float16,float16,0,0.1953173279762268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,64,0,1,float16,fp8,0,0.23152534166971842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,64,0,1,fp8,fp8,0,0.19836799303690592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,64,128,1,fp8,fp8,0,0.26291199525197345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,64,128,1,float16,float16,0,0.20191999276479086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,64,128,1,float16,fp8,0,0.20242132743199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,64,0,1,fp8,fp8,0,0.17202132940292358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,64,0,1,float16,fp8,0,0.19692800442377725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,64,0,1,float16,float16,0,0.1962719957033793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,64,128,1,float16,float16,0,0.20475733280181885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,64,128,1,float16,fp8,0,0.20402133464813232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,64,128,1,fp8,fp8,0,0.2632159988085429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,64,0,1,float16,fp8,0,0.19785600900650024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,64,0,1,fp8,fp8,0,0.17300800482432047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,64,128,1,float16,float16,0,0.20478934049606323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,64,0,1,float16,float16,0,0.19946666558583578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,64,128,1,float16,fp8,0,0.2055199940999349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,64,128,1,fp8,fp8,0,0.26843200127283734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,64,0,1,float16,fp8,0,0.20137067635854086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,64,0,1,fp8,fp8,0,0.17434666554133096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,64,128,1,float16,float16,0,0.2081013321876526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,64,0,1,float16,float16,0,0.20173867543538412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,64,128,1,float16,fp8,0,0.20708266894022623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,64,128,1,fp8,fp8,0,0.2678613265355428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,64,0,1,float16,fp8,0,0.2013066609700521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,64,0,1,fp8,fp8,0,0.1771626671155294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,64,128,1,float16,float16,0,0.1336426635583242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,64,0,1,float16,float16,0,0.14080533385276794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,64,128,1,float16,fp8,0,0.13191999991734824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,64,128,1,fp8,fp8,0,0.16328000028928122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,64,0,1,float16,fp8,0,0.1358560025691986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,64,0,1,fp8,fp8,0,0.1109386682510376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,64,128,1,float16,float16,0,0.11375466982523601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,64,0,1,float16,float16,0,0.108815997838974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,64,0,1,float16,float16,0,0.1090826690196991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,64,128,1,float16,fp8,0,0.11347732941309611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,64,128,1,fp8,fp8,0,0.14897066354751587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,64,0,1,float16,fp8,0,0.10894933342933655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,64,0,1,fp8,fp8,0,0.09709333380063374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,64,128,1,float16,float16,0,0.11364266276359558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,64,128,1,float16,fp8,0,0.11462400356928508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,64,128,1,fp8,fp8,0,0.15033066272735596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,64,0,1,float16,fp8,0,0.10897066195805867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,64,0,1,fp8,fp8,0,0.09756267070770264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,64,128,1,float16,float16,0,0.1146453320980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,64,0,1,float16,float16,0,0.10971732934315999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,64,128,1,float16,fp8,0,0.11541866262753804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,64,128,1,fp8,fp8,0,0.15042133132616678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,64,0,1,float16,fp8,0,0.11016533772150676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,64,0,1,fp8,fp8,0,0.09888533751169841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,64,128,1,float16,float16,0,0.11609599987665813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,64,0,1,float16,float16,0,0.11081600189208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,64,128,1,float16,fp8,0,0.1167733371257782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,64,128,1,fp8,fp8,0,0.15235199530919394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,64,128,1,float16,fp8,0,0.07693866888682048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,64,128,1,fp8,fp8,0,0.09753066301345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,64,0,1,float16,fp8,0,0.11121066411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,64,0,1,fp8,fp8,0,0.10012267033259074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,64,128,1,float16,float16,0,0.07751999795436859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,64,0,1,float16,float16,0,0.06250666578610738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,64,0,1,float16,float16,0,0.07065600156784058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,64,0,1,float16,fp8,0,0.07109333574771881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,64,128,1,float16,float16,0,0.06620799998442332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,64,0,1,fp8,fp8,0,0.06507733464241028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,64,128,1,float16,fp8,0,0.06647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,64,0,1,float16,fp8,0,0.06308799982070923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,64,128,1,fp8,fp8,0,0.0846560001373291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,64,0,1,fp8,fp8,0,0.05502399802207947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,64,128,1,float16,float16,0,0.06659733255704244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,64,0,1,float16,float16,0,0.062447999914487205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,64,128,1,float16,fp8,0,0.06613333523273468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,64,128,1,fp8,fp8,0,0.08535466591517131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,64,0,1,float16,fp8,0,0.0631573349237442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,64,0,1,fp8,fp8,0,0.05648533503214518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,64,128,1,float16,float16,0,0.06718933085600536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,64,0,1,float16,float16,0,0.0637546678384145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,64,128,1,float16,fp8,0,0.06746133168538411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,64,128,1,fp8,fp8,0,0.08592533071835835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,64,0,1,float16,fp8,0,0.06451733410358429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,64,0,1,fp8,fp8,0,0.05736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,64,128,1,float16,float16,0,0.0691786656777064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,64,0,1,float16,float16,0,0.0640533318122228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,64,128,1,float16,fp8,0,0.0682239979505539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,64,128,1,fp8,fp8,0,0.08597333232561748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,64,0,1,float16,fp8,0,0.0639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,64,128,1,fp8,fp8,0,0.061493332187334694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,64,0,1,fp8,fp8,0,0.05958400170008341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,64,128,1,float16,float16,0,0.04674666623274485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,64,0,1,float16,float16,0,0.04450666904449463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,64,128,1,float16,fp8,0,0.04611733555793762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,64,0,1,float16,fp8,0,0.04384533564249674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,64,0,1,fp8,fp8,0,0.03928533444801966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,64,128,1,float16,float16,0,0.04390400151411692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,64,128,1,float16,float16,0,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,64,0,1,float16,float16,0,0.04074133435885111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,64,128,1,float16,fp8,0,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,64,128,1,fp8,fp8,0,0.05327466626962026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,64,0,1,float16,fp8,0,0.04167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,64,0,1,fp8,fp8,0,0.03701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,64,0,1,float16,float16,0,0.04087999959786733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,64,128,1,float16,fp8,0,0.04419200122356415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,64,128,1,fp8,fp8,0,0.0525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,64,0,1,float16,fp8,0,0.0408693328499794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,64,0,1,fp8,fp8,0,0.037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,64,128,1,float16,float16,0,0.04412800073623657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,64,0,1,float16,float16,0,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,64,128,1,float16,fp8,0,0.044138665000597634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,64,0,1,float16,float16,0,0.042170668641726174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,64,128,1,fp8,fp8,0,0.05390933156013489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,64,0,1,float16,fp8,0,0.04144533226887385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,64,128,1,float16,float16,0,0.03585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,64,0,1,fp8,fp8,0,0.037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,64,128,1,float16,float16,0,0.044079999128977455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,64,128,1,float16,fp8,0,0.04455466568470001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,64,128,1,fp8,fp8,0,0.05515733361244202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,64,0,1,float16,fp8,0,0.04261866708596548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,64,128,1,float16,float16,0,0.03347733368476232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,64,0,1,fp8,fp8,0,0.038021333515644073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,64,0,1,float16,float16,0,0.032229334115982056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,64,128,1,float16,fp8,0,0.037317333122094475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,64,128,1,fp8,fp8,0,0.04127999891837438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,64,0,1,float16,fp8,0,0.032672000428040825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,64,0,1,fp8,fp8,0,0.028250666956106823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,64,0,1,float16,float16,0,0.028442665934562683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,64,128,1,float16,fp8,0,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,64,128,1,fp8,fp8,0,0.040106666584809623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,64,0,1,float16,fp8,0,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,64,0,1,fp8,fp8,0,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,64,128,1,float16,float16,0,0.03339733431736628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,64,0,1,float16,float16,0,0.028805332879225414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,64,128,1,float16,fp8,0,0.0334346666932106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,64,128,1,fp8,fp8,0,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,64,0,1,float16,fp8,0,0.028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,64,0,1,fp8,fp8,0,0.02699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,64,128,1,float16,float16,0,0.03433600068092346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,64,0,1,float16,float16,0,0.030261332790056866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,64,128,1,float16,fp8,0,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,64,128,1,fp8,fp8,0,0.04001600046952566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,64,0,1,float16,fp8,0,0.029520000020662945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,64,0,1,fp8,fp8,0,0.02698666602373123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,64,128,1,float16,float16,0,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,64,0,1,float16,float16,0,0.029653333127498627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,64,128,1,float16,fp8,0,0.034629332522551216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,64,128,1,fp8,fp8,0,0.040976000328858696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,64,0,1,float16,fp8,0,0.03189333279927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,64,0,1,fp8,fp8,0,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,64,128,1,float16,float16,0,1.1936266422271729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,64,0,1,float16,float16,0,1.1585973103841145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,64,128,1,float16,fp8,0,1.1927893161773682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,64,0,1,float16,fp8,0,1.1670666535695393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,64,0,1,fp8,fp8,0,1.062725305557251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,64,128,1,float16,float16,0,1.205504020055135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,64,128,1,fp8,fp8,0,1.6385653813680012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,64,0,1,float16,float16,0,1.1730613708496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,64,128,1,float16,fp8,0,1.2030773162841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,64,128,1,fp8,fp8,0,1.6348692576090496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,64,0,1,float16,fp8,0,1.172602653503418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,64,0,1,fp8,fp8,0,1.0770293076833088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,64,128,1,float16,float16,0,1.225610653559367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,64,0,1,float16,float16,0,1.1887679894765217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,64,128,1,float16,fp8,0,1.2189173698425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,64,128,1,fp8,fp8,0,1.6507679621378581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,64,0,1,float16,fp8,0,1.185322682062785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,64,0,1,fp8,fp8,0,1.0892480214436848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,64,128,1,float16,float16,0,1.3013599713643391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,64,0,1,float16,float16,0,1.2713066736857097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,64,128,1,float16,fp8,0,1.2896160284678142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,64,128,1,float16,float16,0,0.8007733027140299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,64,0,1,float16,fp8,0,1.2640159924825032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,64,0,1,fp8,fp8,0,1.10752534866333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,64,128,1,fp8,fp8,0,1.6697866121927898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,64,0,1,float16,float16,0,0.7912106513977051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,64,128,1,float16,fp8,0,0.7725066343943278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,64,128,1,fp8,fp8,0,0.9255680243174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,64,0,1,float16,fp8,0,0.7563199996948242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,64,0,1,fp8,fp8,0,0.6456319888432821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,64,128,1,float16,float16,0,0.6014826695124308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,64,0,1,float16,float16,0,0.5874293247858683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,64,128,1,float16,fp8,0,0.6130293210347494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,64,128,1,fp8,fp8,0,0.8296693166097006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,64,0,1,float16,fp8,0,0.5855786800384521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,64,0,1,fp8,fp8,0,0.543343981107076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,64,128,1,float16,float16,0,0.6110293467839559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,64,0,1,float16,float16,0,0.5963573455810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,64,128,1,float16,fp8,0,0.611786683400472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,64,0,1,fp8,fp8,0,0.5363839864730835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,64,0,1,float16,fp8,0,0.6028213500976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,64,128,1,fp8,fp8,0,0.8319253126780192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,64,128,1,float16,float16,0,0.6292266845703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,64,0,1,float16,float16,0,0.6063199838002523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,64,128,1,float16,fp8,0,0.6198559999465942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,64,128,1,fp8,fp8,0,0.8223199844360352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,64,0,1,float16,fp8,0,0.6059786478678385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,64,0,1,fp8,fp8,0,0.5508213440577189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,64,128,1,float16,float16,0,0.6454933484395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,64,0,1,float16,float16,0,0.6303573449452718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,64,128,1,float16,fp8,0,0.6334933439890543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,64,128,1,fp8,fp8,0,0.8344480196634928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,64,128,1,float16,float16,0,0.41165868441263836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,64,0,1,float16,fp8,0,0.6211146513621012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,64,0,1,float16,float16,0,0.40718400478363037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,64,0,1,fp8,fp8,0,0.5585653384526571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,64,128,1,float16,fp8,0,0.3998826742172241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,64,128,1,fp8,fp8,0,0.47276798884073895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,64,0,1,fp8,fp8,0,0.330074667930603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,64,0,1,float16,fp8,0,0.39186131954193115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,64,128,1,float16,float16,0,0.3119093378384908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,64,0,1,float16,float16,0,0.30586665868759155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,64,128,1,float16,fp8,0,0.3144746621449788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,64,128,1,fp8,fp8,0,0.42128535111745197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,64,0,1,float16,fp8,0,0.3060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,64,0,1,fp8,fp8,0,0.2817973295847575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,64,128,1,float16,float16,0,0.3145066698392232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,64,0,1,float16,float16,0,0.3078239957491557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,64,128,1,float16,fp8,0,0.31623999277750653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,64,0,1,float16,fp8,0,0.3100693424542745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,64,128,1,fp8,fp8,0,0.42161067326863605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,64,0,1,fp8,fp8,0,0.28302399317423504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,64,128,1,float16,float16,0,0.32054932912190753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,64,0,1,float16,float16,0,0.31401065985361737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,64,128,1,float16,fp8,0,0.3184266686439514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,64,128,1,fp8,fp8,0,0.4214506546656291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,64,0,1,float16,fp8,0,0.3134773373603821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,64,0,1,fp8,fp8,0,0.28410667181015015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,64,128,1,float16,float16,0,0.32839999596277875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,64,0,1,float16,float16,0,0.32200000683466595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,64,128,1,float16,fp8,0,0.3280906677246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,64,128,1,fp8,fp8,0,0.42665600776672363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,64,0,1,float16,fp8,0,0.31943466265996295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,64,0,1,fp8,fp8,0,0.28597333033879596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,64,128,1,float16,float16,0,0.21727999051411948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,64,0,1,float16,float16,0,0.216538667678833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,64,128,1,float16,fp8,0,0.21081600586573282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,64,128,1,fp8,fp8,0,0.24506133794784546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,64,0,1,float16,fp8,0,0.20942399899164835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,64,0,1,fp8,fp8,0,0.17812800407409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,64,128,1,float16,float16,0,0.1667520006497701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,64,0,1,float16,float16,0,0.1616320013999939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,64,128,1,float16,fp8,0,0.16634133458137512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,64,128,1,fp8,fp8,0,0.222271998723348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,64,0,1,float16,fp8,0,0.16305599610010782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,64,0,1,fp8,fp8,0,0.15243732929229736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,64,128,1,float16,float16,0,0.16802666584650675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,64,0,1,float16,float16,0,0.16381333271662393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,64,128,1,float16,fp8,0,0.16862932840983072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,64,128,1,fp8,fp8,0,0.22160534063975015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,64,0,1,float16,fp8,0,0.16429866353670755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,64,0,1,fp8,fp8,0,0.15449066956837973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,64,128,1,float16,float16,0,0.17086400588353476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,64,0,1,float16,float16,0,0.1673813263575236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,64,128,1,float16,fp8,0,0.17064533631006876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,64,128,1,fp8,fp8,0,0.2251466711362203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,64,0,1,float16,fp8,0,0.16767466068267822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,64,0,1,fp8,fp8,0,0.1554026703039805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,64,128,1,float16,float16,0,0.17460266749064127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,64,0,1,float16,float16,0,0.1721120079358419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,64,128,1,float16,float16,0,0.12153599659601848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,64,128,1,float16,fp8,0,0.1742453376452128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,64,0,1,float16,float16,0,0.12193600336710612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,64,128,1,fp8,fp8,0,0.2246239980061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,64,0,1,float16,fp8,0,0.17140799760818481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,64,0,1,fp8,fp8,0,0.15574933091799417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,64,128,1,float16,fp8,0,0.12053866187731425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,64,128,1,fp8,fp8,0,0.133050670226415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,64,0,1,float16,fp8,0,0.11843199531237285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,64,0,1,fp8,fp8,0,0.10052800178527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,64,128,1,float16,float16,0,0.0920853316783905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,64,0,1,float16,float16,0,0.09447466333707173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,64,128,1,float16,fp8,0,0.09262933333714803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,64,128,1,fp8,fp8,0,0.12184000015258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,64,0,1,float16,fp8,0,0.09609599908192952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,64,0,1,fp8,fp8,0,0.08713600039482117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,64,128,1,float16,float16,0,0.09300800164540608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,64,0,1,float16,float16,0,0.0950986643632253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,64,128,1,float16,fp8,0,0.09284800291061401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,64,128,1,fp8,fp8,0,0.12213333447774251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,64,0,1,float16,fp8,0,0.09593600034713745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,64,0,1,fp8,fp8,0,0.0881813367207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,64,128,1,float16,float16,0,0.09402133027712505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,64,0,1,float16,float16,0,0.09641599655151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,64,128,1,float16,fp8,0,0.09380799531936646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,64,128,1,fp8,fp8,0,0.12291199962298076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,64,0,1,float16,fp8,0,0.09613866607348125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,64,0,1,fp8,fp8,0,0.08865599830945332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,64,128,1,float16,float16,0,0.096261332432429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,64,0,1,float16,float16,0,0.09828799962997437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,64,128,1,float16,fp8,0,0.09604799747467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,64,0,1,float16,float16,0,0.07195200026035309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,64,128,1,fp8,fp8,0,0.12232533097267151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,64,0,1,float16,fp8,0,0.09854400157928467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,64,0,1,fp8,fp8,0,0.08959999680519104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,64,128,1,float16,float16,0,0.06980800131956737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,64,128,1,float16,fp8,0,0.06725866595904033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,64,128,1,fp8,fp8,0,0.07619733115037282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,64,128,1,fp8,fp8,0,0.06653333206971486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,64,0,1,float16,fp8,0,0.06864533325036366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,64,0,1,fp8,fp8,0,0.058117335041364036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,64,128,1,float16,float16,0,0.0561706672112147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,64,0,1,float16,float16,0,0.05680533250172933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,64,128,1,float16,fp8,0,0.05663999915122986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,64,0,1,float16,fp8,0,0.056789333621660866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,64,0,1,fp8,fp8,0,0.04926399886608124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,64,128,1,float16,float16,0,0.0566240002711614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,64,0,1,float16,float16,0,0.057029331723848976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,64,128,1,float16,fp8,0,0.05670933425426483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,64,128,1,float16,fp8,0,0.05771199862162272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,64,128,1,fp8,fp8,0,0.06814933319886525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,64,128,1,fp8,fp8,0,0.06622933348019917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,64,0,1,float16,fp8,0,0.05682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,64,0,1,fp8,fp8,0,0.04898133377234141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,64,128,1,float16,float16,0,0.057855998476346336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,64,0,1,float16,float16,0,0.057461331288019814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,64,0,1,float16,fp8,0,0.05819199979305267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,64,0,1,fp8,fp8,0,0.05073066552480062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,64,128,1,float16,float16,0,0.058320000767707825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,64,0,1,float16,float16,0,0.05868266522884369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,64,128,1,float16,fp8,0,0.05861333509286245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,64,128,1,fp8,fp8,0,0.07008533179759979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,64,128,1,fp8,fp8,0,0.04740266501903534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,64,0,1,float16,fp8,0,0.05875733494758606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,64,0,1,fp8,fp8,0,0.052757332722345986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,64,128,1,float16,float16,0,0.04085866610209147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,64,0,1,float16,float16,0,0.03957866628964742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,64,128,1,float16,fp8,0,0.04057066639264425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,64,0,1,float16,fp8,0,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,64,0,1,fp8,fp8,0,0.03673599908749262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,64,128,1,float16,float16,0,0.037471999724706016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,64,0,1,float16,float16,0,0.036389333506425224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,64,128,1,float16,fp8,0,0.037589333951473236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,64,128,1,fp8,fp8,0,0.04393066465854645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,64,0,1,float16,fp8,0,0.03641066700220108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,64,0,1,fp8,fp8,0,0.034901333351929985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,64,128,1,float16,float16,0,0.037205333511034645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,64,0,1,float16,float16,0,0.03648533423741659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,64,128,1,float16,fp8,0,0.0376800000667572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,64,128,1,fp8,fp8,0,0.04412800073623657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,64,0,1,float16,fp8,0,0.03665599972009659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,64,0,1,fp8,fp8,0,0.03444266567627589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,64,128,1,float16,float16,0,0.037920000652472176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,64,0,1,float16,float16,0,0.03735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,64,128,1,float16,fp8,0,0.037434667348861694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,64,128,1,fp8,fp8,0,0.04539200166861216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,64,128,1,fp8,fp8,0,0.046869332591692604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,64,0,1,float16,fp8,0,0.03697066754102707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,64,0,1,fp8,fp8,0,0.035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,64,128,1,float16,float16,0,0.03799466788768768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,64,0,1,float16,float16,0,0.037258667250474296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,64,128,1,float16,fp8,0,0.03845333307981491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,64,0,1,float16,fp8,0,0.03742400060097376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,64,0,1,fp8,fp8,0,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,64,128,1,float16,float16,0,0.02996266633272171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,64,0,1,float16,float16,0,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,64,128,1,float16,fp8,0,0.02974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,64,128,1,fp8,fp8,0,0.0351200004418691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,64,0,1,float16,fp8,0,0.02935466667016347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,64,0,1,fp8,fp8,0,0.028042666614055634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,64,128,1,float16,float16,0,0.027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,64,0,1,float16,float16,0,0.02682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,64,0,1,float16,float16,0,0.026906666656335194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,64,128,1,float16,fp8,0,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,64,128,1,fp8,fp8,0,0.03320533285538355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,64,0,1,float16,fp8,0,0.02682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,64,0,1,fp8,fp8,0,0.026362667481104534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,64,128,1,float16,float16,0,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,64,128,1,float16,fp8,0,0.02741333345572154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,64,128,1,fp8,fp8,0,0.03382933388153712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,64,0,1,float16,fp8,0,0.02657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,64,0,1,fp8,fp8,0,0.026464000344276428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,64,128,1,float16,float16,0,0.02757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,64,0,1,float16,float16,0,0.026543999711672466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,64,128,1,float16,fp8,0,0.02775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,64,128,1,fp8,fp8,0,0.03389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,64,0,1,float16,fp8,0,0.0271519993742307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,64,0,1,fp8,fp8,0,0.026447998980681103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,64,128,1,float16,float16,0,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,64,0,1,float16,float16,0,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,64,128,1,float16,fp8,0,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,64,128,1,fp8,fp8,0,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,64,0,1,float16,fp8,0,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,64,0,1,fp8,fp8,0,0.027119999130566914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,64,128,1,float16,float16,0,0.019706666469573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,64,0,1,float16,float16,0,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,64,128,1,float16,fp8,0,0.020037333170572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,64,128,1,fp8,fp8,0,0.023376000424226124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,64,0,1,float16,fp8,0,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,64,0,1,fp8,fp8,0,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,64,128,1,float16,float16,0,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,64,0,1,float16,float16,0,0.018373332917690277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,64,128,1,float16,fp8,0,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,64,128,1,fp8,fp8,0,0.023344000180562336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,64,0,1,float16,fp8,0,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,64,0,1,fp8,fp8,0,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,64,128,1,float16,float16,0,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,64,0,1,float16,float16,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,64,128,1,float16,fp8,0,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,64,128,1,float16,fp8,0,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,64,128,1,fp8,fp8,0,0.02334933231274287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,64,0,1,float16,fp8,0,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,64,0,1,fp8,fp8,0,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,64,128,1,float16,float16,0,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,64,0,1,float16,float16,0,0.01860800012946129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,64,0,1,float16,float16,0,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,64,128,1,fp8,fp8,0,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,64,0,1,float16,fp8,0,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,64,0,1,fp8,fp8,0,0.02024000013868014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,64,128,1,float16,float16,0,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,64,128,1,float16,fp8,0,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,64,128,1,fp8,fp8,0,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,64,0,1,float16,fp8,0,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,64,0,1,fp8,fp8,0,0.01998399943113327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,64,128,1,float16,float16,0,0.44328534603118896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,64,0,1,float16,float16,0,0.4429226716359456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,64,128,1,float16,fp8,0,0.44555731614430744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,64,128,1,fp8,fp8,0,0.5334453185399374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,64,0,1,float16,fp8,0,0.4424373308817546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,64,0,1,fp8,fp8,0,0.5321013530095419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,64,128,1,float16,float16,0,0.45390931765238446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,64,0,1,float16,float16,0,0.4543893337249756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,64,128,1,float16,fp8,0,0.45289599895477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,64,128,1,fp8,fp8,0,0.535103996594747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,64,0,1,float16,fp8,0,0.45263465245564777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,64,0,1,fp8,fp8,0,0.5364053249359131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,64,128,1,float16,float16,0,0.46457600593566895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,64,0,1,float16,float16,0,0.4635680119196574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,64,128,1,float16,fp8,0,0.46297601858774823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,64,128,1,fp8,fp8,0,0.5389120181401571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,64,0,1,float16,fp8,0,0.4633280038833618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,64,0,1,fp8,fp8,0,0.5382826725641886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,64,128,1,float16,float16,0,0.4817119836807251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,64,0,1,float16,float16,0,0.48057599862416583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,64,128,1,float16,fp8,0,0.47627735137939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,64,128,1,fp8,fp8,0,0.554693341255188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,64,0,1,float16,fp8,0,0.47570133209228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,64,0,1,fp8,fp8,0,0.5545493364334106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,64,128,1,float16,float16,0,0.34039465586344403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,64,0,1,float16,float16,0,0.339354674021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,64,128,1,float16,fp8,0,0.3264000018437703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,64,128,1,fp8,fp8,0,0.32939199606577557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,64,0,1,float16,fp8,0,0.32579733928044635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,64,128,1,float16,float16,0,0.23058666785558066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,64,0,1,fp8,fp8,0,0.3285973270734151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,64,0,1,float16,float16,0,0.23106133937835693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,64,128,1,float16,fp8,0,0.2318506638209025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,64,128,1,fp8,fp8,0,0.2769013245900472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,64,0,1,float16,fp8,0,0.23009065786997476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,64,0,1,fp8,fp8,0,0.27827733755111694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,64,128,1,float16,float16,0,0.23272534211476645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,64,0,1,float16,float16,0,0.2337119976679484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,64,0,1,fp8,fp8,0,0.27797333399454754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,64,128,1,float16,fp8,0,0.23293334245681763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,64,128,1,fp8,fp8,0,0.2797759970029195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,64,0,1,float16,fp8,0,0.23407999674479166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,64,128,1,float16,float16,0,0.23730667432149252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,64,0,1,float16,float16,0,0.23697600762049356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,64,128,1,float16,fp8,0,0.23737066984176636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,64,128,1,fp8,fp8,0,0.2825919985771179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,64,0,1,float16,fp8,0,0.23699732621510824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,64,0,1,fp8,fp8,0,0.28134934107462567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,64,128,1,float16,float16,0,0.24572267134984335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,64,0,1,float16,float16,0,0.24433600902557373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,64,128,1,float16,fp8,0,0.2424373428026835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,64,128,1,fp8,fp8,0,0.2847626606623332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,64,0,1,float16,fp8,0,0.24390933911005655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,64,128,1,float16,float16,0,0.17866667111714682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,64,0,1,fp8,fp8,0,0.2836373249689738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,64,0,1,float16,float16,0,0.1789813240369161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,64,128,1,float16,fp8,0,0.1718506614367167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,64,128,1,fp8,fp8,0,0.17618666092554727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,64,0,1,float16,fp8,0,0.1718239982922872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,64,0,1,fp8,fp8,0,0.17547200123469034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,64,128,1,float16,float16,0,0.12378133336702983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,64,0,1,float16,float16,0,0.12332800030708313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,64,128,1,float16,fp8,0,0.12468799948692322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,64,128,1,fp8,fp8,0,0.1513599952061971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,64,0,1,float16,fp8,0,0.1239306628704071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,64,0,1,fp8,fp8,0,0.15135467052459717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,64,128,1,float16,float16,0,0.12564266721407572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,64,128,1,fp8,fp8,0,0.15346133708953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,64,0,1,float16,float16,0,0.12489599982897441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,64,128,1,float16,fp8,0,0.12546666463216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,64,0,1,float16,fp8,0,0.12533332904179892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,64,0,1,fp8,fp8,0,0.15313067038853964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,64,128,1,float16,float16,0,0.1276639997959137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,64,0,1,float16,fp8,0,0.12761066357294717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,64,0,1,float16,float16,0,0.12813867131868997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,64,128,1,float16,fp8,0,0.12800533572832742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,64,128,1,fp8,fp8,0,0.15434666474660239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,64,0,1,fp8,fp8,0,0.1548853317896525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,64,128,1,float16,float16,0,0.13010666767756143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,64,0,1,float16,float16,0,0.13058132926623026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,64,128,1,float16,fp8,0,0.12964266538619995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,64,128,1,fp8,fp8,0,0.15532799561818442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,64,0,1,float16,fp8,0,0.13057600458463034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,64,0,1,fp8,fp8,0,0.15474133690198263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,64,128,1,float16,float16,0,0.09645332892735799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,64,0,1,float16,float16,0,0.0960693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,64,128,1,float16,fp8,0,0.09331199526786804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,64,128,1,fp8,fp8,0,0.0988159974416097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,64,0,1,float16,fp8,0,0.09333866834640503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,64,0,1,fp8,fp8,0,0.09915733337402344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,64,128,1,fp8,fp8,0,0.08687466382980347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,64,128,1,float16,float16,0,0.06952000161012013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,64,0,1,float16,float16,0,0.0703306645154953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,64,128,1,float16,fp8,0,0.06972800195217133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,64,128,1,float16,fp8,0,0.0711359977722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,64,0,1,float16,fp8,0,0.06965333223342896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,64,128,1,fp8,fp8,0,0.0863200028737386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,64,0,1,fp8,fp8,0,0.08675199747085571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,64,128,1,float16,float16,0,0.0710453341404597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,64,0,1,float16,float16,0,0.07062399884064992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,64,0,1,float16,fp8,0,0.07123200098673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,64,0,1,fp8,fp8,0,0.08678399523099263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,64,0,1,float16,fp8,0,0.07195200026035309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,64,128,1,float16,float16,0,0.07133866846561432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,64,0,1,float16,float16,0,0.070783997575442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,64,128,1,float16,fp8,0,0.07178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,64,128,1,fp8,fp8,0,0.0879200001557668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,64,128,1,fp8,fp8,0,0.08939733107884724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,64,0,1,fp8,fp8,0,0.08797333637873332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,64,128,1,float16,float16,0,0.07340799768765767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,64,0,1,float16,float16,0,0.07281599938869476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,64,128,1,float16,fp8,0,0.07307200133800507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,64,0,1,float16,fp8,0,0.07320533196131389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,64,0,1,fp8,fp8,0,0.08914666374524434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,64,0,1,fp8,fp8,0,0.057162667314211525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,64,128,1,float16,float16,0,0.0537013312180837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,64,0,1,float16,float16,0,0.05378133555253347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,64,128,1,float16,fp8,0,0.05195199946562449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,64,128,1,fp8,fp8,0,0.057317331433296204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,64,0,1,float16,fp8,0,0.05171200136343638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,64,128,1,float16,float16,0,0.040048000713189445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,64,0,1,float16,float16,0,0.04055999964475632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,64,128,1,float16,fp8,0,0.04060266663630804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,64,128,1,fp8,fp8,0,0.04948266843954722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,64,0,1,float16,fp8,0,0.04083200047413508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,64,0,1,fp8,fp8,0,0.05041599770387014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,64,128,1,float16,float16,0,0.04060266663630804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,64,0,1,float16,float16,0,0.04030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,64,128,1,float16,fp8,0,0.04117333392302195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,64,128,1,fp8,fp8,0,0.050111999114354454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,64,0,1,float16,fp8,0,0.04062933226426443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,64,0,1,fp8,fp8,0,0.04949333270390829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,64,0,1,fp8,fp8,0,0.05151999990145365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,64,128,1,float16,float16,0,0.041482667128245033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,64,0,1,float16,float16,0,0.04165866722663244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,64,128,1,float16,fp8,0,0.041493333876132965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,64,128,1,fp8,fp8,0,0.050623998045921326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,64,0,1,float16,fp8,0,0.04181333382924398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,64,128,1,float16,float16,0,0.04262933135032654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,64,0,1,float16,float16,0,0.042303999265034996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,64,128,1,float16,fp8,0,0.042223999897638954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,64,128,1,fp8,fp8,0,0.051738664507865906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,64,0,1,float16,fp8,0,0.042490666111310325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,64,0,1,fp8,fp8,0,0.052239999175071716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,64,128,1,float16,float16,0,0.03033066789309184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,64,0,1,float16,float16,0,0.029658667743206024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,64,128,1,float16,fp8,0,0.030063999195893604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,64,128,1,fp8,fp8,0,0.036848001182079315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,64,0,1,float16,fp8,0,0.029845332105954487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,64,0,1,fp8,fp8,0,0.03696533292531967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,64,128,1,float16,float16,0,0.027045334378878277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,64,0,1,float16,float16,0,0.027024000883102417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,64,128,1,float16,fp8,0,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,64,128,1,fp8,fp8,0,0.03389333436886469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,64,0,1,float16,fp8,0,0.02741866558790207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,64,0,1,fp8,fp8,0,0.034304000437259674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,64,128,1,float16,float16,0,0.026922665536403656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,64,0,1,float16,float16,0,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,64,128,1,float16,fp8,0,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,64,128,1,fp8,fp8,0,0.03462400039037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,64,0,1,float16,fp8,0,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,64,0,1,fp8,fp8,0,0.03420799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,64,128,1,float16,float16,0,0.026698666314284008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,64,0,1,float16,float16,0,0.027082666754722595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,64,128,1,float16,fp8,0,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,64,128,1,fp8,fp8,0,0.034741332133611046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,64,0,1,float16,fp8,0,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,64,0,1,fp8,fp8,0,0.03440533330043157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,64,128,1,float16,float16,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,64,0,1,float16,float16,0,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,64,128,1,float16,fp8,0,0.027952000498771667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,64,128,1,fp8,fp8,0,0.03555200000603994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,64,128,1,float16,fp8,0,0.023034666975339253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,64,0,1,float16,fp8,0,0.028330666323502857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,64,0,1,fp8,fp8,0,0.03562133262554804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,64,0,1,fp8,fp8,0,0.028501334289709728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,64,128,1,float16,float16,0,0.02239999920129776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,64,0,1,float16,float16,0,0.02275199939807256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,64,128,1,float16,fp8,0,0.02080533280968666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,64,128,1,fp8,fp8,0,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,64,0,1,float16,fp8,0,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,64,128,1,float16,float16,0,0.02060266708334287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,64,0,1,float16,float16,0,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,64,128,1,fp8,fp8,0,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,64,0,1,float16,fp8,0,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,64,0,1,fp8,fp8,0,0.025968000292778015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,64,128,1,float16,float16,0,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,64,0,1,float16,float16,0,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,64,128,1,float16,fp8,0,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,64,128,1,fp8,fp8,0,0.026586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,64,0,1,float16,fp8,0,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,64,0,1,fp8,fp8,0,0.026250667870044708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,64,128,1,float16,float16,0,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,64,0,1,float16,float16,0,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,64,128,1,float16,fp8,0,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,64,128,1,fp8,fp8,0,0.026149332523345947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,64,0,1,float16,fp8,0,0.021738665799299877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,64,128,1,float16,float16,0,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,64,0,1,fp8,fp8,0,0.026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,64,0,1,float16,float16,0,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,64,128,1,float16,fp8,0,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,64,128,1,fp8,fp8,0,0.02672533442576726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,64,0,1,float16,fp8,0,0.021583999196688335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,64,0,1,fp8,fp8,0,0.027050666511058807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,64,128,1,float16,float16,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,64,0,1,float16,float16,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,64,128,1,float16,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,64,128,1,fp8,fp8,0,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,64,0,1,float16,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,64,0,1,fp8,fp8,0,0.020581333587567013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,64,128,1,float16,float16,0,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,64,128,1,float16,float16,0,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,64,0,1,float16,float16,0,0.01605333387851715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,64,128,1,float16,fp8,0,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,64,128,1,fp8,fp8,0,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,64,0,1,float16,fp8,0,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,64,0,1,fp8,fp8,0,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,64,0,1,float16,float16,0,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,64,128,1,float16,fp8,0,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,64,128,1,fp8,fp8,0,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,64,0,1,float16,fp8,0,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,64,0,1,fp8,fp8,0,0.020096000283956528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,64,128,1,float16,float16,0,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,64,0,1,float16,float16,0,0.01579733317097028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,64,128,1,float16,fp8,0,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,64,128,1,fp8,fp8,0,0.019402666638294857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,64,0,1,float16,fp8,0,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,64,0,1,fp8,fp8,0,0.019808000574509304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,64,0,1,fp8,fp8,0,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,64,128,1,float16,float16,0,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,64,0,1,float16,float16,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,64,128,1,float16,fp8,0,0.016309333344300587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,64,128,1,fp8,fp8,0,0.019685332973798115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,64,0,1,float16,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,64,128,1,float16,float16,0,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,64,0,1,float16,float16,0,0.014042666802803675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,64,128,1,float16,fp8,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,64,128,1,fp8,fp8,0,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,64,0,1,float16,fp8,0,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,64,0,1,fp8,fp8,0,0.019498666127522785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,64,128,1,float16,float16,0,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,64,0,1,float16,float16,0,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,64,128,1,float16,fp8,0,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,64,128,1,fp8,fp8,0,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,64,0,1,float16,fp8,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,64,0,1,fp8,fp8,0,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,64,128,1,float16,float16,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,64,0,1,float16,float16,0,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,64,128,1,float16,fp8,0,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,64,128,1,fp8,fp8,0,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,64,0,1,float16,fp8,0,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,64,0,1,fp8,fp8,0,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,64,128,1,float16,float16,0,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,64,0,1,float16,float16,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,64,128,1,float16,fp8,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,64,128,1,fp8,fp8,0,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,64,0,1,float16,fp8,0,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,64,0,1,fp8,fp8,0,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,64,0,1,float16,fp8,0,0.014287999520699183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,64,128,1,float16,float16,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,64,0,1,float16,float16,0,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,64,128,1,float16,fp8,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,64,128,1,fp8,fp8,0,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,64,0,1,fp8,fp8,0,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,64,128,1,float16,float16,0,0.21678932507832846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,64,0,1,float16,float16,0,0.21818133195241293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,64,128,1,float16,fp8,0,0.21818667650222778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,64,0,1,float16,fp8,0,0.21887467304865518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,64,128,1,fp8,fp8,0,0.37445334593455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,64,128,1,float16,float16,0,0.21975467602411905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,64,0,1,fp8,fp8,0,0.3752426703770955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,64,0,1,float16,float16,0,0.21946134169896445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,64,128,1,float16,fp8,0,0.21955200036366782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,64,128,1,fp8,fp8,0,0.3787786563237508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,64,0,1,float16,fp8,0,0.22099733352661133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,64,0,1,fp8,fp8,0,0.37570667266845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,64,128,1,float16,float16,0,0.22470933198928833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,64,0,1,float16,float16,0,0.22459733486175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,64,0,1,float16,fp8,0,0.22393065690994263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,64,128,1,float16,fp8,0,0.2238666613896688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,64,128,1,fp8,fp8,0,0.3789386749267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,64,128,1,float16,float16,0,0.2294506629308065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,64,0,1,fp8,fp8,0,0.3795146544774373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,64,0,1,float16,float16,0,0.2306293249130249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,64,128,1,float16,fp8,0,0.22911999622980753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,64,128,1,float16,fp8,0,0.1683786710103353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,64,0,1,float16,fp8,0,0.22874132792154947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,64,0,1,fp8,fp8,0,0.3855946858723958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,64,128,1,fp8,fp8,0,0.38159998257954914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,64,0,1,float16,fp8,0,0.16764267285664877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,64,128,1,float16,float16,0,0.1169706682364146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,64,128,1,float16,float16,0,0.17562667528788248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,64,0,1,float16,float16,0,0.17525333166122437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,64,128,1,fp8,fp8,0,0.22363199790318808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,64,0,1,fp8,fp8,0,0.2242400050163269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,64,0,1,fp8,fp8,0,0.20099733273188272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,64,0,1,float16,float16,0,0.11685867110888164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,64,128,1,float16,fp8,0,0.11710932850837708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,64,128,1,fp8,fp8,0,0.20114666223526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,64,0,1,float16,fp8,0,0.11736533045768738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,64,0,1,float16,float16,0,0.11867733796437581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,64,128,1,float16,float16,0,0.11998400092124939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,64,128,1,float16,fp8,0,0.11884799599647522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,64,128,1,fp8,fp8,0,0.2013546625773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,64,0,1,float16,fp8,0,0.1181599994500478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,64,0,1,fp8,fp8,0,0.20078933238983154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,64,128,1,float16,float16,0,0.12158399820327759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,64,0,1,float16,float16,0,0.12108266353607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,64,128,1,float16,fp8,0,0.12110400199890137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,64,0,1,float16,fp8,0,0.12138666709264119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,64,128,1,fp8,fp8,0,0.20317333936691284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,64,128,1,float16,float16,0,0.12211199601491292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,64,0,1,fp8,fp8,0,0.20270933707555136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,64,0,1,float16,float16,0,0.12292266885439555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,64,128,1,float16,fp8,0,0.1227946678797404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,64,128,1,fp8,fp8,0,0.20482667287190756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,64,0,1,float16,fp8,0,0.12185066938400269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,64,128,1,float16,float16,0,0.09185600280761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,64,0,1,fp8,fp8,0,0.2044586737950643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,64,0,1,float16,float16,0,0.09267733494440715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,64,128,1,float16,fp8,0,0.08947199583053589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,64,128,1,float16,fp8,0,0.0658240020275116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,64,128,1,fp8,fp8,0,0.12105600039164226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,64,128,1,fp8,fp8,0,0.11178132891654968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,64,0,1,float16,fp8,0,0.08948799967765808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,64,0,1,fp8,fp8,0,0.12181867162386577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,64,128,1,float16,float16,0,0.06492266555627187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,64,0,1,float16,float16,0,0.06489600241184235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,64,0,1,float16,fp8,0,0.06492800017197926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,64,0,1,fp8,fp8,0,0.11058132847150166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,64,128,1,float16,float16,0,0.06570666531721751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,64,0,1,float16,float16,0,0.06549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,64,128,1,float16,fp8,0,0.06564799944559734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,64,128,1,fp8,fp8,0,0.11064533392588298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,64,0,1,float16,fp8,0,0.06554133196671803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,64,0,1,fp8,fp8,0,0.11172266801198323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,64,128,1,float16,float16,0,0.0662720004717509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,64,0,1,float16,float16,0,0.06668266654014587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,64,128,1,float16,fp8,0,0.06621866424878438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,64,128,1,fp8,fp8,0,0.11321066816647847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,64,0,1,float16,fp8,0,0.06639466683069865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,64,0,1,fp8,fp8,0,0.11196266611417134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,64,128,1,float16,float16,0,0.06855466465155284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,64,0,1,float16,float16,0,0.06979733208815257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,64,128,1,float16,fp8,0,0.06894933183987935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,64,128,1,fp8,fp8,0,0.11313066879908244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,64,0,1,float16,fp8,0,0.06875733534495036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,64,0,1,fp8,fp8,0,0.11426132917404175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,64,128,1,float16,float16,0,0.04976533353328705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,64,0,1,float16,float16,0,0.050479998191197716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,64,128,1,float16,fp8,0,0.047557334105173744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,64,128,1,fp8,fp8,0,0.07030400137106578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,64,0,1,float16,fp8,0,0.04712533454100291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,64,0,1,fp8,fp8,0,0.06927466889222463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,64,128,1,float16,float16,0,0.03828799972931544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,64,0,1,float16,fp8,0,0.038746667404969536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,64,0,1,float16,float16,0,0.03763733307520548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,64,128,1,float16,fp8,0,0.03867733230193456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,64,128,1,fp8,fp8,0,0.06406933565934499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,64,0,1,fp8,fp8,0,0.06189866860707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,64,128,1,float16,float16,0,0.0383840004603068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,64,0,1,float16,float16,0,0.03843733419974645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,64,128,1,float16,fp8,0,0.038506666819254555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,64,0,1,float16,float16,0,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,64,128,1,float16,fp8,0,0.039317332208156586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,64,128,1,fp8,fp8,0,0.0625493327776591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,64,0,1,float16,fp8,0,0.03851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,64,0,1,fp8,fp8,0,0.06214400132497152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,64,128,1,float16,float16,0,0.039503999054431915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,64,128,1,fp8,fp8,0,0.0647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,64,128,1,float16,fp8,0,0.04015466570854187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,64,0,1,float16,fp8,0,0.03925866633653641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,64,0,1,fp8,fp8,0,0.06317333380381267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,64,0,1,fp8,fp8,0,0.06497066716353099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,64,128,1,float16,float16,0,0.03942399968703588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,64,0,1,float16,float16,0,0.040634666879971824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,64,128,1,fp8,fp8,0,0.0645653357108434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,64,0,1,float16,fp8,0,0.04026666780312856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,64,0,1,float16,fp8,0,0.027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,64,128,1,float16,float16,0,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,64,0,1,float16,float16,0,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,64,128,1,float16,fp8,0,0.028064000109831493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,64,128,1,fp8,fp8,0,0.04139200101296107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,64,0,1,fp8,fp8,0,0.04095466683308283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,64,128,1,float16,float16,0,0.02491733431816101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,64,0,1,float16,float16,0,0.02430933217207591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,64,128,1,float16,fp8,0,0.024773334463437397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,64,128,1,fp8,fp8,0,0.039434666434923805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,64,0,1,float16,fp8,0,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,64,128,1,fp8,fp8,0,0.03869866579771042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,64,0,1,fp8,fp8,0,0.03879466652870178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,64,128,1,float16,float16,0,0.024842667082945507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,64,0,1,float16,float16,0,0.02463999887307485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,64,128,1,float16,fp8,0,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,64,0,1,float16,fp8,0,0.024885334074497223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,64,0,1,fp8,fp8,0,0.03886933376391729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,64,128,1,float16,float16,0,0.02465066562096278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,64,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,64,128,1,float16,fp8,0,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,64,128,1,fp8,fp8,0,0.03946666667858759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,64,0,1,float16,fp8,0,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,64,0,1,fp8,fp8,0,0.039317332208156586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,64,0,1,fp8,fp8,0,0.04036800066630045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,64,128,1,float16,float16,0,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,64,0,1,float16,float16,0,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,64,128,1,float16,fp8,0,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,64,128,1,fp8,fp8,0,0.04020266731580099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,64,0,1,float16,fp8,0,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,64,128,1,float16,float16,0,0.01978133370478948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,64,0,1,float16,float16,0,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,64,128,1,fp8,fp8,0,0.030192000170548756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,64,128,1,float16,fp8,0,0.020784000555674236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,64,0,1,float16,fp8,0,0.02059200033545494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,64,0,1,fp8,fp8,0,0.029861333469549816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,64,128,1,float16,float16,0,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,64,0,1,float16,float16,0,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,64,128,1,float16,fp8,0,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,64,128,1,fp8,fp8,0,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,64,0,1,float16,fp8,0,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,64,0,1,float16,fp8,0,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,64,0,1,fp8,fp8,0,0.02914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,64,128,1,float16,float16,0,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,64,0,1,float16,float16,0,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,64,128,1,float16,fp8,0,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,64,128,1,fp8,fp8,0,0.02829866607983907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,64,0,1,fp8,fp8,0,0.028837333122889202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,64,128,1,float16,float16,0,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,64,0,1,float16,float16,0,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,64,0,1,float16,float16,0,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,64,128,1,float16,fp8,0,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,64,128,1,fp8,fp8,0,0.028789333999156952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,64,0,1,float16,fp8,0,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,64,0,1,fp8,fp8,0,0.02924799919128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,64,128,1,float16,float16,0,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,64,128,1,float16,fp8,0,0.019813333948453266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,64,128,1,fp8,fp8,0,0.029205332199732464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,64,0,1,float16,fp8,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,64,0,1,fp8,fp8,0,0.02939733366171519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,64,128,1,float16,float16,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,64,0,1,float16,float16,0,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,64,128,1,float16,fp8,0,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,64,128,1,fp8,fp8,0,0.02399466683467229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,64,0,1,float16,fp8,0,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,64,0,1,fp8,fp8,0,0.02367466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,64,128,1,float16,float16,0,0.014394666999578476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,64,0,1,float16,float16,0,0.014485333114862442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,64,128,1,float16,fp8,0,0.014650666465361914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,64,128,1,fp8,fp8,0,0.02293866624434789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,64,0,1,float16,fp8,0,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,64,0,1,fp8,fp8,0,0.022629333039124806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,64,128,1,float16,float16,0,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,64,0,1,float16,float16,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,64,128,1,float16,fp8,0,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,64,128,1,fp8,fp8,0,0.022229333718617756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,64,0,1,float16,fp8,0,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,64,0,1,fp8,fp8,0,0.022202665607134502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,64,128,1,float16,float16,0,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,64,0,1,float16,float16,0,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,64,128,1,float16,fp8,0,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,64,128,1,fp8,fp8,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,64,0,1,float16,fp8,0,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,64,0,1,fp8,fp8,0,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,64,128,1,float16,float16,0,0.01458666721979777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,64,0,1,float16,float16,0,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,64,128,1,float16,fp8,0,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,64,128,1,fp8,fp8,0,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,64,0,1,float16,fp8,0,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,64,0,1,fp8,fp8,0,0.02274133265018463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,64,128,1,float16,float16,0,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,64,0,1,float16,float16,0,0.012565333396196365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,64,128,1,float16,float16,0,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,64,128,1,float16,fp8,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,64,128,1,fp8,fp8,0,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,64,0,1,float16,fp8,0,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,64,0,1,fp8,fp8,0,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,64,0,1,float16,float16,0,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,64,128,1,float16,fp8,0,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,64,128,1,fp8,fp8,0,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,64,0,1,float16,fp8,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,64,0,1,fp8,fp8,0,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,64,128,1,float16,float16,0,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,64,0,1,float16,float16,0,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,64,128,1,float16,fp8,0,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,64,128,1,fp8,fp8,0,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,64,0,1,float16,fp8,0,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,64,0,1,fp8,fp8,0,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,64,128,1,float16,float16,0,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,64,0,1,float16,float16,0,0.01257066677014033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,64,128,1,float16,fp8,0,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,64,128,1,fp8,fp8,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,64,0,1,float16,fp8,0,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,64,0,1,fp8,fp8,0,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,64,128,1,float16,float16,0,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,64,0,1,float16,float16,0,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,64,128,1,float16,fp8,0,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,64,128,1,fp8,fp8,0,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,64,0,1,float16,fp8,0,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,64,0,1,fp8,fp8,0,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,64,128,1,float16,float16,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,64,0,1,float16,float16,0,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,64,128,1,float16,fp8,0,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,64,128,1,fp8,fp8,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,64,0,1,float16,fp8,0,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,64,0,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,64,128,1,float16,float16,0,0.012085333466529846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,64,0,1,float16,float16,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,64,128,1,float16,fp8,0,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,64,128,1,fp8,fp8,0,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,64,0,1,float16,fp8,0,0.01239466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,64,0,1,fp8,fp8,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,64,128,1,float16,float16,0,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,64,0,1,float16,float16,0,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,64,128,1,float16,fp8,0,0.01250133290886879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,64,128,1,fp8,fp8,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,64,0,1,float16,fp8,0,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,64,0,1,fp8,fp8,0,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,64,128,1,float16,float16,0,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,64,0,1,float16,float16,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,64,128,1,float16,fp8,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,64,128,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,64,0,1,float16,fp8,0,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,64,0,1,fp8,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,64,128,1,float16,float16,0,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,64,0,1,float16,float16,0,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,64,128,1,float16,fp8,0,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,64,128,1,fp8,fp8,0,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,64,0,1,float16,fp8,0,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,64,0,1,fp8,fp8,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,64,128,1,float16,float16,0,0.12772267063458762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,64,0,1,float16,float16,0,0.1281706690788269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,64,128,1,float16,fp8,0,0.12799466649691263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,64,0,1,float16,fp8,0,0.1288320024808248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,64,128,1,fp8,fp8,0,0.31547733147939044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,64,128,1,float16,float16,0,0.12928533554077148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,64,0,1,fp8,fp8,0,0.3146933317184448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,64,0,1,float16,float16,0,0.12946666280428568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,64,128,1,float16,float16,0,0.13372266292572021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,64,128,1,float16,fp8,0,0.1292746663093567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,64,0,1,float16,fp8,0,0.12958932916323343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,64,128,1,fp8,fp8,0,0.31587199370066327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,64,0,1,fp8,fp8,0,0.31520533561706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,64,0,1,float16,float16,0,0.13358400265375772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,64,128,1,float16,fp8,0,0.13313600420951843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,64,0,1,float16,fp8,0,0.13296000162760416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,64,128,1,fp8,fp8,0,0.3176746765772502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,64,0,1,fp8,fp8,0,0.3179840048154195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,64,128,1,float16,fp8,0,0.13663466771443686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,64,128,1,float16,float16,0,0.13758933544158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,64,0,1,float16,fp8,0,0.1356106698513031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,64,0,1,float16,float16,0,0.13701333602269491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,64,128,1,fp8,fp8,0,0.31842132409413654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,64,128,1,float16,float16,0,0.0934879978497823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,64,0,1,float16,float16,0,0.0941439966360728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,64,0,1,fp8,fp8,0,0.3188106616338094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,64,0,1,fp8,fp8,0,0.18126400311787924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,64,128,1,float16,fp8,0,0.090037335952123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,64,128,1,fp8,fp8,0,0.18132799863815308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,64,0,1,float16,fp8,0,0.09054932991663615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,64,128,1,float16,float16,0,0.07006399830182393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,64,0,1,float16,float16,0,0.06993600229422252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,64,128,1,float16,fp8,0,0.06964266796906789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,64,128,1,fp8,fp8,0,0.16742932796478271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,64,0,1,float16,fp8,0,0.06976533432801564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,64,128,1,float16,float16,0,0.06985066831111908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,64,0,1,fp8,fp8,0,0.16878400246302286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,64,0,1,float16,float16,0,0.07072533170382182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,64,128,1,float16,fp8,0,0.07113066812356313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,64,128,1,fp8,fp8,0,0.16806399822235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,64,0,1,float16,fp8,0,0.06972266733646393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,64,0,1,fp8,fp8,0,0.16878400246302286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,64,0,1,float16,fp8,0,0.07150400181611379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,64,128,1,float16,float16,0,0.07101333141326904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,64,0,1,float16,float16,0,0.07179200152556102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,64,128,1,float16,fp8,0,0.07215466598669688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,64,128,1,fp8,fp8,0,0.1699893275896708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,64,0,1,fp8,fp8,0,0.16898133357365927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,64,128,1,float16,float16,0,0.0734559992949168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,64,0,1,float16,float16,0,0.07449066638946533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,64,128,1,float16,fp8,0,0.07283199826876323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,64,128,1,fp8,fp8,0,0.17091200749079385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,64,0,1,float16,fp8,0,0.0728000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,64,0,1,fp8,fp8,0,0.17042134205500284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,64,128,1,float16,float16,0,0.050613333781560264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,64,0,1,float16,float16,0,0.05032533407211304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,64,128,1,float16,fp8,0,0.04844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,64,128,1,fp8,fp8,0,0.10016533732414246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,64,0,1,float16,fp8,0,0.047482664386431374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,64,0,1,fp8,fp8,0,0.09954667091369629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,64,128,1,float16,float16,0,0.038506666819254555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,64,0,1,float16,float16,0,0.03845333307981491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,64,128,1,float16,fp8,0,0.03850133220354716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,64,128,1,fp8,fp8,0,0.09074667096138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,64,0,1,float16,fp8,0,0.03932799895604452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,64,0,1,fp8,fp8,0,0.09286933143933614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,64,128,1,float16,float16,0,0.03851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,64,0,1,float16,float16,0,0.038773333032925926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,64,128,1,float16,fp8,0,0.03867200016975403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,64,128,1,fp8,fp8,0,0.0913759966691335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,64,0,1,float16,fp8,0,0.038560000558694206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,64,0,1,fp8,fp8,0,0.09184533357620239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,64,128,1,float16,float16,0,0.04043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,64,0,1,float16,float16,0,0.04012266546487808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,64,128,1,float16,fp8,0,0.039546666045983635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,64,128,1,fp8,fp8,0,0.09293867150942485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,64,0,1,float16,fp8,0,0.03988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,64,0,1,fp8,fp8,0,0.09347732861836751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,64,128,1,float16,float16,0,0.04043200115362803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,64,0,1,float16,float16,0,0.04041599979003271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,64,128,1,float16,fp8,0,0.041434665520985924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,64,128,1,fp8,fp8,0,0.09429867068926494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,64,0,1,float16,fp8,0,0.03982399900754293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,64,0,1,fp8,fp8,0,0.09377066294352214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,64,128,1,float16,float16,0,0.027104000250498455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,64,0,1,float16,float16,0,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,64,128,1,float16,fp8,0,0.026538667579491932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,64,128,1,fp8,fp8,0,0.05715733269850413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,64,0,1,float16,fp8,0,0.029674666623274486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,64,0,1,float16,fp8,0,0.025562666356563568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,64,0,1,fp8,fp8,0,0.0577706644932429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,64,0,1,float16,float16,0,0.02571200082699458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,64,128,1,float16,float16,0,0.025583999852339428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,64,128,1,float16,fp8,0,0.02565866708755493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,64,0,1,float16,float16,0,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,64,128,1,float16,fp8,0,0.025834667185942333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,64,128,1,fp8,fp8,0,0.05530133346716563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,64,0,1,fp8,fp8,0,0.05621333420276642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,64,128,1,float16,float16,0,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,64,128,1,fp8,fp8,0,0.05458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,64,0,1,float16,fp8,0,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,64,0,1,fp8,fp8,0,0.05464000006516775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,64,128,1,float16,float16,0,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,64,0,1,float16,float16,0,0.026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,64,128,1,float16,fp8,0,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,64,128,1,fp8,fp8,0,0.054789334535598755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,64,0,1,float16,fp8,0,0.026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,64,0,1,float16,fp8,0,0.026176000634829204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,64,0,1,fp8,fp8,0,0.05541333556175232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,64,128,1,float16,float16,0,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,64,0,1,float16,float16,0,0.02593066543340683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,64,128,1,float16,fp8,0,0.02651199946800868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,64,128,1,fp8,fp8,0,0.055813332398732506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,64,0,1,fp8,fp8,0,0.055402666330337524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,64,128,1,float16,float16,0,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,64,0,1,float16,float16,0,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,64,128,1,float16,fp8,0,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,64,128,1,fp8,fp8,0,0.0364533339937528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,64,0,1,float16,fp8,0,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,64,0,1,fp8,fp8,0,0.035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,64,128,1,float16,float16,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,64,0,1,float16,float16,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,64,128,1,float16,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,64,0,1,float16,float16,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,64,128,1,fp8,fp8,0,0.035045333206653595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,64,0,1,float16,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,64,0,1,fp8,fp8,0,0.035045333206653595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,64,128,1,float16,float16,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,64,128,1,float16,fp8,0,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,64,128,1,fp8,fp8,0,0.03522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,64,0,1,float16,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,64,0,1,fp8,fp8,0,0.03549866626660029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,64,128,1,float16,float16,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,64,0,1,float16,float16,0,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,64,128,1,float16,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,64,128,1,fp8,fp8,0,0.03470933437347412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,64,0,1,float16,fp8,0,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,64,0,1,fp8,fp8,0,0.03536533315976461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,64,128,1,float16,float16,0,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,64,0,1,float16,float16,0,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,64,128,1,float16,fp8,0,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,64,128,1,fp8,fp8,0,0.03577066709597906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,64,0,1,float16,fp8,0,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,64,0,1,fp8,fp8,0,0.03573866685231527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,64,128,1,float16,float16,0,0.014138666292031607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,64,0,1,fp8,fp8,0,0.026591998835404713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,64,0,1,float16,float16,0,0.013978666315476099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,64,128,1,float16,fp8,0,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,64,128,1,fp8,fp8,0,0.02685333291689555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,64,0,1,float16,fp8,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,64,128,1,float16,float16,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,64,0,1,float16,float16,0,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,64,128,1,float16,fp8,0,0.014426667243242264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,64,128,1,fp8,fp8,0,0.025754667818546295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,64,0,1,float16,fp8,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,64,0,1,fp8,fp8,0,0.02587733417749405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,64,128,1,float16,float16,0,0.014074667046467463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,64,0,1,float16,float16,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,64,128,1,float16,fp8,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,64,128,1,fp8,fp8,0,0.026165333886941273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,64,0,1,float16,fp8,0,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,64,0,1,fp8,fp8,0,0.026346666117509205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,64,128,1,float16,float16,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,64,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,64,128,1,float16,fp8,0,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,64,128,1,fp8,fp8,0,0.025850666066010792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,64,128,1,float16,fp8,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,64,0,1,float16,fp8,0,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,64,0,1,fp8,fp8,0,0.026373334228992462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,64,128,1,float16,float16,0,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,64,0,1,float16,float16,0,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,64,128,1,fp8,fp8,0,0.026954665780067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,64,0,1,float16,fp8,0,0.01444799949725469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,64,0,1,fp8,fp8,0,0.026543999711672466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,64,128,1,float16,float16,0,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,64,0,1,float16,float16,0,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,64,128,1,float16,fp8,0,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,64,128,1,fp8,fp8,0,0.022282667458057404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,64,0,1,float16,fp8,0,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,64,0,1,fp8,fp8,0,0.022277332842350006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,64,128,1,float16,float16,0,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,64,0,1,float16,float16,0,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,64,128,1,float16,fp8,0,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,64,128,1,fp8,fp8,0,0.0215786670645078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,64,0,1,float16,fp8,0,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,64,0,1,fp8,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,64,0,1,fp8,fp8,0,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,64,128,1,float16,float16,0,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,64,0,1,float16,float16,0,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,64,128,1,float16,fp8,0,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,64,128,1,fp8,fp8,0,0.021477334201335907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,64,0,1,float16,fp8,0,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,64,0,1,fp8,fp8,0,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,64,128,1,float16,float16,0,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,64,0,1,float16,float16,0,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,64,128,1,float16,fp8,0,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,64,128,1,fp8,fp8,0,0.021984001000722248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,64,0,1,float16,fp8,0,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,64,0,1,fp8,fp8,0,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,64,128,1,float16,float16,0,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,64,0,1,float16,float16,0,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,64,128,1,float16,fp8,0,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,64,128,1,fp8,fp8,0,0.022117334107557934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,64,0,1,float16,fp8,0,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,64,128,1,float16,float16,0,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,64,0,1,float16,float16,0,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,64,128,1,float16,fp8,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,64,128,1,fp8,fp8,0,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,64,0,1,float16,fp8,0,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,64,0,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,64,128,1,float16,float16,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,64,0,1,float16,float16,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,64,128,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,64,128,1,fp8,fp8,0,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,64,0,1,float16,fp8,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,64,128,1,fp8,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,64,0,1,fp8,fp8,0,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,64,128,1,float16,float16,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,64,0,1,float16,float16,0,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,64,0,1,float16,float16,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,64,128,1,float16,fp8,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,64,0,1,float16,fp8,0,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,64,0,1,fp8,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,64,128,1,float16,float16,0,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,64,128,1,float16,fp8,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,64,128,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,64,128,1,float16,fp8,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,64,0,1,float16,fp8,0,0.011920000116030375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,64,0,1,fp8,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,64,0,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,64,128,1,float16,float16,0,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,64,0,1,float16,float16,0,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,64,128,1,fp8,fp8,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,64,0,1,float16,fp8,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,64,128,1,float16,float16,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,64,0,1,float16,float16,0,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,64,128,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,64,128,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,64,0,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,64,0,1,fp8,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,64,128,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,64,0,1,float16,float16,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,64,128,1,float16,fp8,0,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,64,128,1,fp8,fp8,0,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,64,0,1,float16,fp8,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,64,0,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,64,128,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,64,0,1,float16,float16,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,64,128,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,64,128,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,64,0,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,64,0,1,fp8,fp8,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,64,128,1,float16,float16,0,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,64,0,1,fp8,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,64,0,1,float16,float16,0,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,64,128,1,float16,fp8,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,64,128,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,64,128,1,float16,fp8,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,64,0,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,64,128,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,64,0,1,float16,fp8,0,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,64,128,1,float16,float16,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,64,0,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,64,0,1,float16,fp8,0,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,64,128,1,float16,float16,0,0.09344533085823059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,64,0,1,float16,float16,0,0.0937653382619222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,64,128,1,float16,fp8,0,0.0937439997990926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,64,0,1,float16,fp8,0,0.09406399726867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,64,128,1,fp8,fp8,0,0.2844799955685933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,64,128,1,float16,float16,0,0.09435199697812398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,64,0,1,fp8,fp8,0,0.2842186689376831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,64,0,1,float16,float16,0,0.09437333544095357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,64,128,1,float16,fp8,0,0.09427199761072795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,64,128,1,float16,float16,0,0.0951039989789327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,64,0,1,float16,fp8,0,0.09429867068926494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,64,128,1,fp8,fp8,0,0.28406933943430585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,64,0,1,fp8,fp8,0,0.2850026686986287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,64,0,1,float16,float16,0,0.09532800316810608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,64,128,1,float16,fp8,0,0.09518399834632874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,64,128,1,fp8,fp8,0,0.2848373254140218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,64,0,1,float16,fp8,0,0.09539199868837993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,64,128,1,float16,fp8,0,0.09702400366465251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,64,128,1,float16,float16,0,0.09638399879137675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,64,0,1,fp8,fp8,0,0.28520532449086505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,64,0,1,float16,float16,0,0.0962506632010142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,64,0,1,float16,fp8,0,0.09684266646703084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,64,128,1,fp8,fp8,0,0.2865973313649495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,64,128,1,float16,float16,0,0.059024001161257424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,64,0,1,float16,float16,0,0.06084266801675161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,64,0,1,fp8,fp8,0,0.2859039902687073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,64,128,1,float16,fp8,0,0.05586666862169901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,64,128,1,fp8,fp8,0,0.15774400035540262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,64,0,1,float16,fp8,0,0.05585599939028422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,64,128,1,float16,float16,0,0.05045866469542185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,64,0,1,fp8,fp8,0,0.15801599621772766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,64,0,1,float16,float16,0,0.050810664892196655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,64,128,1,float16,fp8,0,0.05045866469542185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,64,0,1,float16,fp8,0,0.050250664353370667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,64,128,1,fp8,fp8,0,0.14918399850527445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,64,0,1,fp8,fp8,0,0.15106667081514993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,64,128,1,float16,float16,0,0.05082133412361145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,64,0,1,float16,float16,0,0.05065600077311198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,64,128,1,float16,fp8,0,0.050944000482559204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,64,128,1,fp8,fp8,0,0.15026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,64,0,1,float16,fp8,0,0.050885334610939026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,64,0,1,fp8,fp8,0,0.14989333351453146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,64,128,1,float16,float16,0,0.0516533354918162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,64,0,1,float16,float16,0,0.051856001218159996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,64,128,1,float16,fp8,0,0.05160533388455709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,64,128,1,fp8,fp8,0,0.15078399578730264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,64,0,1,float16,fp8,0,0.0518453319867452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,64,0,1,fp8,fp8,0,0.15093866984049478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,64,128,1,float16,float16,0,0.05239466826121012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,64,0,1,float16,float16,0,0.05248000224431356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,64,128,1,float16,fp8,0,0.05177066723505656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,64,128,1,fp8,fp8,0,0.15146133303642273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,64,0,1,float16,fp8,0,0.052570665876070656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,64,128,1,fp8,fp8,0,0.08753599723180135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,64,0,1,float16,fp8,0,0.031770666440327965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,64,128,1,float16,float16,0,0.03230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,64,0,1,fp8,fp8,0,0.15230400363604227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,64,0,1,float16,float16,0,0.03209600100914637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,64,128,1,float16,fp8,0,0.03151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,64,128,1,float16,fp8,0,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,64,128,1,fp8,fp8,0,0.08476266264915466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,64,0,1,float16,fp8,0,0.031162666777769726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,64,0,1,fp8,fp8,0,0.08574933807055156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,64,128,1,float16,float16,0,0.031471999982992806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,64,0,1,float16,float16,0,0.031125334401925404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,64,0,1,fp8,fp8,0,0.08497599760691325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,64,128,1,float16,float16,0,0.03161066770553589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,64,0,1,float16,float16,0,0.03169066707293192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,64,128,1,float16,fp8,0,0.031386665999889374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,64,0,1,float16,float16,0,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,64,128,1,fp8,fp8,0,0.08530666430791219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,64,0,1,float16,fp8,0,0.031685332457224526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,64,0,1,fp8,fp8,0,0.08358933528264363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,64,128,1,float16,float16,0,0.031541332602500916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,64,128,1,float16,fp8,0,0.03209600100914637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,64,128,1,fp8,fp8,0,0.08373333017031352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,64,0,1,float16,fp8,0,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,64,0,1,fp8,fp8,0,0.08459200461705525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,64,128,1,float16,float16,0,0.03223466624816259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,64,0,1,fp8,fp8,0,0.0848586658636729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,64,0,1,float16,float16,0,0.0317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,64,128,1,float16,fp8,0,0.03200533241033554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,64,128,1,fp8,fp8,0,0.0855466624101003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,64,0,1,float16,fp8,0,0.03186133255561193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,64,128,1,float16,float16,0,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,64,0,1,float16,float16,0,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,64,128,1,float16,fp8,0,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,64,128,1,fp8,fp8,0,0.05131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,64,0,1,float16,fp8,0,0.02181333303451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,64,0,1,fp8,fp8,0,0.05239999790986379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,64,128,1,float16,float16,0,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,64,0,1,fp8,fp8,0,0.05100800096988678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,64,0,1,float16,float16,0,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,64,128,1,float16,fp8,0,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,64,128,1,fp8,fp8,0,0.05023466547330221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,64,0,1,float16,fp8,0,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,64,128,1,float16,float16,0,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,64,0,1,fp8,fp8,0,0.05110399921735128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,64,0,1,float16,float16,0,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,64,128,1,float16,fp8,0,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,64,128,1,fp8,fp8,0,0.051130667328834534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,64,0,1,float16,fp8,0,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,64,128,1,float16,float16,0,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,64,0,1,float16,float16,0,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,64,128,1,float16,fp8,0,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,64,128,1,fp8,fp8,0,0.050570666790008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,64,0,1,float16,fp8,0,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,64,0,1,fp8,fp8,0,0.05036266644795736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,64,128,1,float16,float16,0,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,64,0,1,float16,float16,0,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,64,128,1,float16,fp8,0,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,64,128,1,fp8,fp8,0,0.050383999943733215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,64,0,1,float16,fp8,0,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,64,0,1,fp8,fp8,0,0.05137600004673004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,64,128,1,float16,float16,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,64,0,1,float16,float16,0,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,64,128,1,float16,fp8,0,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,64,128,1,fp8,fp8,0,0.03307733436425527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,64,0,1,float16,fp8,0,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,64,128,1,float16,float16,0,0.014597332725922266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,64,0,1,fp8,fp8,0,0.03316266586383184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,64,0,1,float16,float16,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,64,0,1,float16,float16,0,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,64,128,1,float16,fp8,0,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,64,128,1,fp8,fp8,0,0.03183466692765554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,64,0,1,float16,fp8,0,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,64,0,1,fp8,fp8,0,0.03275733441114426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,64,128,1,float16,float16,0,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,64,128,1,float16,float16,0,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,64,128,1,float16,fp8,0,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,64,128,1,fp8,fp8,0,0.032218667368094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,64,128,1,fp8,fp8,0,0.032261334359645844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,64,0,1,float16,fp8,0,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,64,128,1,float16,float16,0,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,64,0,1,fp8,fp8,0,0.03229333211978277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,64,0,1,float16,float16,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,64,128,1,float16,fp8,0,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,64,0,1,float16,fp8,0,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,64,0,1,fp8,fp8,0,0.03219733387231827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,64,0,1,float16,float16,0,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,64,128,1,float16,fp8,0,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,64,128,1,fp8,fp8,0,0.0327360009153684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,64,0,1,float16,fp8,0,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,64,0,1,fp8,fp8,0,0.03259733319282532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,64,128,1,float16,float16,0,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,64,0,1,float16,float16,0,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,64,128,1,float16,fp8,0,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,64,128,1,float16,fp8,0,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,64,128,1,fp8,fp8,0,0.02550933261712392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,64,0,1,float16,fp8,0,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,64,0,1,fp8,fp8,0,0.025888000925381977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,64,128,1,float16,float16,0,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,64,0,1,float16,float16,0,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,64,128,1,fp8,fp8,0,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,64,0,1,float16,fp8,0,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,64,0,1,fp8,fp8,0,0.02584533393383026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,64,128,1,float16,float16,0,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,64,0,1,float16,float16,0,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,64,128,1,float16,fp8,0,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,64,128,1,fp8,fp8,0,0.02499733368555705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,64,0,1,float16,fp8,0,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,64,0,1,fp8,fp8,0,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,64,128,1,float16,float16,0,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,64,0,1,float16,float16,0,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,64,128,1,float16,fp8,0,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,64,128,1,fp8,fp8,0,0.025055999557177227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,64,0,1,float16,fp8,0,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,64,128,1,fp8,fp8,0,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,64,0,1,fp8,fp8,0,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,64,128,1,float16,float16,0,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,64,0,1,float16,float16,0,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,64,128,1,float16,fp8,0,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,64,0,1,float16,fp8,0,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,64,0,1,fp8,fp8,0,0.025050667424996693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,64,128,1,float16,float16,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,64,0,1,float16,float16,0,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,64,128,1,float16,fp8,0,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,64,128,1,fp8,fp8,0,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,64,0,1,float16,fp8,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,64,0,1,fp8,fp8,0,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,64,128,1,float16,float16,0,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,64,0,1,float16,float16,0,0.011946666985750198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,64,128,1,float16,fp8,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,64,128,1,fp8,fp8,0,0.020687999824682873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,64,0,1,float16,fp8,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,64,0,1,fp8,fp8,0,0.020421333611011505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,64,128,1,float16,float16,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,64,0,1,float16,float16,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,64,128,1,float16,fp8,0,0.012378666549921036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,64,128,1,fp8,fp8,0,0.020848001043001812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,64,0,1,float16,fp8,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,64,0,1,fp8,fp8,0,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,64,0,1,float16,fp8,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,64,128,1,float16,float16,0,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,64,0,1,float16,float16,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,64,128,1,float16,fp8,0,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,64,128,1,fp8,fp8,0,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,64,0,1,fp8,fp8,0,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,64,128,1,float16,float16,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,64,0,1,float16,float16,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,64,128,1,float16,fp8,0,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,64,128,1,float16,fp8,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,64,128,1,fp8,fp8,0,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,64,0,1,float16,fp8,0,0.012416000167528788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,64,0,1,fp8,fp8,0,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,64,128,1,float16,float16,0,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,64,0,1,float16,float16,0,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,64,128,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,64,0,1,float16,fp8,0,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,64,0,1,fp8,fp8,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,64,128,1,float16,float16,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,64,0,1,float16,float16,0,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,64,128,1,float16,fp8,0,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,64,128,1,fp8,fp8,0,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,64,0,1,float16,fp8,0,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,64,0,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,64,128,1,float16,float16,0,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,64,0,1,float16,float16,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,64,128,1,float16,fp8,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,64,128,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,64,0,1,float16,fp8,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,64,0,1,fp8,fp8,0,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,64,128,1,float16,float16,0,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,64,128,1,float16,float16,0,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,64,0,1,float16,float16,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,64,128,1,float16,fp8,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,64,128,1,fp8,fp8,0,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,64,0,1,float16,fp8,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,64,0,1,fp8,fp8,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,64,0,1,float16,float16,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,64,128,1,float16,fp8,0,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,64,128,1,fp8,fp8,0,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,64,128,1,fp8,fp8,0,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,64,0,1,float16,fp8,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,64,0,1,float16,fp8,0,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,64,0,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,64,128,1,float16,float16,0,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,64,0,1,float16,float16,0,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,64,128,1,float16,fp8,0,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,64,0,1,fp8,fp8,0,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,64,128,1,float16,float16,0,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,64,0,1,float16,float16,0,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,64,128,1,float16,fp8,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,64,128,1,fp8,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,64,128,1,fp8,fp8,0,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,64,0,1,float16,fp8,0,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,64,0,1,fp8,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,64,128,1,float16,float16,0,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,64,0,1,float16,float16,0,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,64,128,1,float16,fp8,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,64,0,1,float16,fp8,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,64,0,1,fp8,fp8,0,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,64,128,1,float16,float16,0,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,64,0,1,float16,float16,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,64,128,1,float16,fp8,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,64,128,1,fp8,fp8,0,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,64,0,1,float16,fp8,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,64,0,1,fp8,fp8,0,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,64,128,1,float16,float16,0,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,64,0,1,float16,float16,0,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,64,128,1,float16,fp8,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,64,128,1,fp8,fp8,0,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,64,0,1,float16,fp8,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,64,0,1,fp8,fp8,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,64,128,1,float16,float16,0,2.8951892852783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,64,128,1,float16,fp8,0,2.867722511291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,64,128,1,float16,float16,0,3.0154987970987954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,64,128,1,fp8,fp8,0,3.6986185709635415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,64,128,1,float16,fp8,0,2.9956000645955405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,64,128,1,fp8,fp8,0,3.8121814727783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,64,128,1,float16,float16,0,3.039386749267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,64,128,1,float16,fp8,0,3.012410799662272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,64,0,1,fp8,fp8,0,17.071210225423176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,64,0,1,float16,float16,0,18.85433069864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,64,0,1,float16,fp8,0,18.677941640218098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,64,128,1,fp8,fp8,0,3.8150132497151694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,64,0,1,float16,float16,0,19.147226969401043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,64,0,1,float16,fp8,0,18.94578679402669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,64,128,1,float16,float16,0,3.0786558787027993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,64,0,1,fp8,fp8,0,17.129610697428387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,64,128,1,float16,fp8,0,3.051386515299479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,64,128,1,fp8,fp8,0,3.8450400034586587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,64,0,1,float16,float16,0,18.908480326334637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,64,128,1,float16,float16,0,1.7052906354268391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,64,128,1,float16,fp8,0,1.6773120562235515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,64,128,1,fp8,fp8,0,2.1361172993977866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,64,0,1,fp8,fp8,0,19.45890172322591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,64,0,1,float16,fp8,0,21.890777587890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,64,0,1,float16,float16,0,10.649898529052734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,64,0,1,float16,float16,0,19.068618774414062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,64,128,1,float16,float16,0,1.4537919362386067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,64,128,1,float16,fp8,0,1.4198506673177083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,64,0,1,fp8,fp8,0,9.140362421671549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,64,0,1,float16,fp8,0,10.857674916585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,64,128,1,fp8,fp8,0,1.8684906959533691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,64,0,1,fp8,fp8,0,19.054117838541668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,64,128,1,float16,float16,0,1.4973279635111492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,64,0,1,float16,fp8,0,19.625818888346355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,64,128,1,float16,fp8,0,1.477882703145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,64,128,1,fp8,fp8,0,1.9401920636494954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,64,128,1,float16,float16,0,1.503551959991455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,64,0,1,float16,float16,0,9.479978561401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,64,128,1,float16,fp8,0,1.4876960118611653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,64,0,1,fp8,fp8,0,8.595237096150717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,64,0,1,float16,fp8,0,9.562864303588867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,64,128,1,fp8,fp8,0,1.950266679128011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,64,0,1,float16,float16,0,9.737546920776367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,64,128,1,float16,float16,0,1.5464213689168294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,64,0,1,fp8,fp8,0,8.673157374064127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,64,0,1,float16,fp8,0,9.546581268310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,64,128,1,float16,fp8,0,1.523189385732015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,64,128,1,fp8,fp8,0,1.9812320073445637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,64,128,1,float16,float16,0,0.8482453028361002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,64,0,1,float16,float16,0,9.565690358479818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,64,128,1,float16,fp8,0,0.8444586594899496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,64,128,1,fp8,fp8,0,1.0969866911570232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,64,0,1,fp8,fp8,0,8.686223983764648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,64,0,1,float16,fp8,0,9.479114532470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,64,128,1,float16,float16,0,0.7614879608154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,64,0,1,float16,float16,0,5.02286942799886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,64,0,1,float16,float16,0,9.607173283894857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,64,128,1,float16,fp8,0,0.7564906279246012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,64,128,1,fp8,fp8,0,1.009552001953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,64,0,1,fp8,fp8,0,4.5010560353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,64,0,1,fp8,fp8,0,8.72871462504069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,64,0,1,float16,fp8,0,5.02460257212321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,64,128,1,float16,float16,0,0.7871466477711996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,64,0,1,float16,fp8,0,10.493733088175455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,64,0,1,float16,float16,0,4.83296012878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,64,128,1,float16,fp8,0,0.7645386854807535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,64,128,1,fp8,fp8,0,0.9993440310160319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,64,128,1,float16,float16,0,0.7732799847920736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,64,0,1,fp8,fp8,0,4.387258529663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,64,128,1,float16,fp8,0,0.7705173492431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,64,0,1,float16,fp8,0,5.064533233642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,64,128,1,fp8,fp8,0,0.9997013409932455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,64,128,1,float16,float16,0,0.8044906457265218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,64,0,1,float16,float16,0,4.9728959401448565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,64,0,1,float16,fp8,0,5.0555572509765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,64,0,1,fp8,fp8,0,4.388330777486165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,64,128,1,float16,fp8,0,0.7921120325724283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,64,0,1,float16,float16,0,4.978373209635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,64,128,1,fp8,fp8,0,1.0221280256907146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,64,128,1,float16,float16,0,0.5497706731160482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,64,0,1,fp8,fp8,0,4.354405403137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,64,128,1,float16,fp8,0,0.5505706469217936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,64,0,1,float16,fp8,0,4.976949373881022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,64,128,1,fp8,fp8,0,0.6643626689910889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,64,128,1,float16,float16,0,0.5574880043665568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,64,0,1,float16,float16,0,5.254624048868815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,64,0,1,float16,float16,0,2.658991972605387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,64,0,1,fp8,fp8,0,4.383989334106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,64,128,1,float16,fp8,0,0.5596160093943278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,64,0,1,fp8,fp8,0,2.3991519610087075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,64,0,1,float16,fp8,0,2.7081867853800454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,64,0,1,float16,fp8,0,4.880586624145508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,64,128,1,fp8,fp8,0,0.6644906600316366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,64,128,1,float16,float16,0,0.5499733289082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,64,128,1,float16,fp8,0,0.5497546593348185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,64,0,1,float16,float16,0,2.5978506406148276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,64,128,1,fp8,fp8,0,0.6644586722056071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,64,128,1,float16,float16,0,0.5496426820755005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,64,0,1,fp8,fp8,0,2.4116533597310386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,64,0,1,float16,fp8,0,2.8025598526000977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,64,0,1,float16,float16,0,2.637104034423828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,64,128,1,float16,fp8,0,0.549343983332316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,64,128,1,fp8,fp8,0,0.6662186781565348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,64,0,1,fp8,fp8,0,2.3792853355407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,64,0,1,float16,fp8,0,2.602191925048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,64,128,1,float16,float16,0,0.5488959948221842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,64,0,1,float16,float16,0,2.63592529296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,64,128,1,float16,fp8,0,0.5477813482284546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,64,128,1,fp8,fp8,0,0.6639573176701864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,64,0,1,float16,fp8,0,2.5976692835489907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,64,0,1,fp8,fp8,0,2.3924105962117515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,64,0,1,float16,float16,0,2.600320021311442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,64,0,1,fp8,fp8,0,2.392853260040283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,64,0,1,float16,fp8,0,2.601029396057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,64,128,1,float16,float16,0,2.1427146593729653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,64,128,1,float16,fp8,0,2.1445120175679526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,64,128,1,fp8,fp8,0,2.775024096171061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,64,128,1,float16,float16,0,2.2609119415283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,64,128,1,float16,fp8,0,2.2533547083536782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,64,128,1,fp8,fp8,0,2.8678080240885415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,64,128,1,float16,float16,0,2.278442700703939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,64,128,1,float16,fp8,0,2.2609386444091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,64,0,1,float16,float16,0,11.063120524088541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,64,0,1,fp8,fp8,0,10.000111897786459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,64,0,1,float16,fp8,0,10.980560302734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,64,0,1,float16,float16,0,11.171722412109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,64,128,1,fp8,fp8,0,2.8619092305501304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,64,0,1,fp8,fp8,0,10.065733591715494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,64,0,1,float16,fp8,0,11.065050760904947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,64,128,1,float16,float16,0,2.3056693077087402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,64,128,1,float16,fp8,0,2.2781173388163247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,64,0,1,float16,float16,0,11.080832163492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,64,128,1,fp8,fp8,0,2.9064693450927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,64,128,1,float16,float16,0,1.2627360026041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,64,128,1,float16,fp8,0,1.2257386843363445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,64,128,1,fp8,fp8,0,1.5998133023579915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,64,0,1,fp8,fp8,0,10.109962463378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,64,0,1,float16,fp8,0,11.595994313557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,64,128,1,float16,float16,0,1.0684160391489665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,64,0,1,float16,float16,0,5.696282704671224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,64,0,1,float16,float16,0,11.19711430867513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,64,128,1,float16,fp8,0,1.0515413284301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,64,0,1,fp8,fp8,0,5.228495915730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,64,0,1,float16,fp8,0,5.838111877441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,64,128,1,fp8,fp8,0,1.4025440216064453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,64,0,1,fp8,fp8,0,10.152575810750326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,64,0,1,float16,fp8,0,11.692000071207682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,64,128,1,float16,float16,0,1.0739093621571858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,64,128,1,float16,fp8,0,1.0709173679351807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,64,128,1,fp8,fp8,0,1.4107413291931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,64,128,1,float16,float16,0,1.083136002222697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,64,0,1,float16,float16,0,6.1038665771484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,64,128,1,float16,fp8,0,1.1106026967366536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,64,0,1,fp8,fp8,0,5.061296145121257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,64,0,1,float16,fp8,0,6.0394242604573565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,64,128,1,fp8,fp8,0,1.4875787099202473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,64,0,1,float16,float16,0,5.800954818725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,64,0,1,fp8,fp8,0,5.045839945475261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,64,128,1,float16,float16,0,1.132362683614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,64,0,1,float16,fp8,0,5.949104309082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,64,128,1,float16,fp8,0,1.1449600060780842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,64,0,1,float16,float16,0,6.062309265136719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,64,128,1,fp8,fp8,0,1.4519306818644206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,64,128,1,float16,float16,0,0.6480693419774374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,64,128,1,float16,fp8,0,0.6309333244959513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,64,128,1,fp8,fp8,0,0.841808001200358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,64,0,1,fp8,fp8,0,5.338864008585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,64,0,1,float16,fp8,0,5.497690836588542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,64,128,1,float16,float16,0,0.5868746836980184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,64,0,1,float16,float16,0,3.134762763977051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,64,0,1,float16,float16,0,5.644112269083659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,64,128,1,float16,fp8,0,0.5764426787694296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,64,0,1,float16,fp8,0,2.890714645385742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,64,0,1,fp8,fp8,0,2.7223145167032876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,64,128,1,fp8,fp8,0,0.8271093368530273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,64,0,1,fp8,fp8,0,5.259605407714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,64,0,1,float16,fp8,0,6.0370133717854815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,64,128,1,float16,float16,0,0.6392533381779989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,64,128,1,float16,fp8,0,0.5814719994862875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,64,0,1,float16,float16,0,3.120533307393392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,64,128,1,fp8,fp8,0,0.7582133611043295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,64,128,1,float16,float16,0,0.5890880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,64,0,1,float16,fp8,0,2.9778238932291665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,64,128,1,float16,fp8,0,0.5826773246129354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,64,0,1,fp8,fp8,0,2.5912960370381675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,64,128,1,fp8,fp8,0,0.7614293098449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,64,0,1,float16,float16,0,2.827546755472819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,64,0,1,float16,fp8,0,2.8297545115152993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,64,0,1,fp8,fp8,0,2.6030933062235513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,64,128,1,float16,float16,0,0.5960586468378702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,64,0,1,float16,float16,0,2.8383305867513022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,64,128,1,float16,fp8,0,0.5934240023295084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,64,128,1,fp8,fp8,0,0.7701173623402914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,64,128,1,float16,float16,0,0.4184533357620239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,64,0,1,float16,fp8,0,2.9958292643229165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,64,0,1,fp8,fp8,0,2.5998667081197104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,64,0,1,float16,float16,0,2.8486932118733725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,64,128,1,float16,fp8,0,0.41894400119781494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,64,128,1,fp8,fp8,0,0.5159146785736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,64,0,1,float16,float16,0,1.5717066129048665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,64,128,1,float16,float16,0,0.4179893334706624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,64,0,1,float16,fp8,0,2.8531360626220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,64,0,1,fp8,fp8,0,2.60699733098348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,64,128,1,float16,fp8,0,0.4209546645482381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,64,0,1,float16,fp8,0,1.5612799326578777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,64,0,1,fp8,fp8,0,1.3869706789652507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,64,128,1,fp8,fp8,0,0.5068159898122152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,64,128,1,float16,float16,0,0.4209226767222087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,64,0,1,float16,float16,0,1.5632373491923015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,64,128,1,float16,fp8,0,0.41874667008717853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,64,128,1,fp8,fp8,0,0.5058240095774332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,64,0,1,fp8,fp8,0,1.3765546480814617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,64,0,1,float16,fp8,0,1.6063253084818523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,64,128,1,float16,float16,0,0.4187519947687785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,64,0,1,float16,float16,0,1.6035146713256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,64,128,1,float16,fp8,0,0.4159466822942098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,64,128,1,fp8,fp8,0,0.5144799947738647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,64,0,1,fp8,fp8,0,1.3903999328613281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,64,0,1,float16,fp8,0,1.5616854031880696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,64,0,1,float16,float16,0,1.5627840360005696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,64,128,1,float16,float16,0,0.41553600629170734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,64,0,1,float16,fp8,0,1.5540159543355305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,64,128,1,float16,fp8,0,0.41887466112772626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,64,128,1,fp8,fp8,0,0.5086186726888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,64,0,1,fp8,fp8,0,1.4095199902852376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,64,0,1,float16,float16,0,1.5645972887674968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,64,0,1,float16,fp8,0,1.56386137008667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,64,0,1,fp8,fp8,0,1.3799039522806804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,64,128,1,float16,float16,0,1.802026589711507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,64,128,1,float16,fp8,0,1.7548373540242512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,64,128,1,fp8,fp8,0,2.348128000895182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,64,128,1,float16,float16,0,1.8714027404785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,64,128,1,float16,fp8,0,1.895477294921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,64,128,1,fp8,fp8,0,2.3910667101542153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,64,128,1,float16,float16,0,1.8963467280069988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,64,0,1,float16,float16,0,7.795653025309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,64,0,1,fp8,fp8,0,7.144405364990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,64,128,1,float16,fp8,0,1.8783520062764485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,64,0,1,float16,fp8,0,7.808624267578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,64,0,1,float16,float16,0,7.860677083333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,64,0,1,fp8,fp8,0,7.225039800008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,64,128,1,fp8,fp8,0,2.4044480323791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,64,128,1,float16,float16,0,1.925439993540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,64,0,1,float16,fp8,0,8.199509302775065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,64,128,1,float16,fp8,0,1.8873759905497234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,64,0,1,float16,float16,0,7.931456247965495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,64,128,1,fp8,fp8,0,2.425439993540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,64,128,1,float16,float16,0,1.0367519855499268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,64,128,1,float16,fp8,0,1.0100586414337158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,64,128,1,fp8,fp8,0,1.3261600335439045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,64,0,1,fp8,fp8,0,7.216880162556966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,64,0,1,float16,fp8,0,7.931205113728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,64,0,1,float16,float16,0,4.070837338765462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,64,128,1,float16,float16,0,0.8799786567687988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,64,0,1,float16,float16,0,8.048896153767904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,64,0,1,fp8,fp8,0,7.279082616170247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,64,128,1,float16,fp8,0,0.8815093040466309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,64,0,1,float16,fp8,0,4.236517270406087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,64,128,1,fp8,fp8,0,1.1464320023854573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,64,0,1,float16,fp8,0,8.036266962687174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,64,0,1,fp8,fp8,0,3.7750186920166016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,64,128,1,float16,float16,0,0.9199413458506266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,64,0,1,float16,float16,0,4.16262944539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,64,128,1,float16,fp8,0,0.8868107000986735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,64,128,1,fp8,fp8,0,1.1811892986297607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,64,128,1,float16,float16,0,0.9343093236287435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,64,128,1,float16,fp8,0,0.8905759652455648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,64,0,1,fp8,fp8,0,3.59660275777181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,64,0,1,float16,fp8,0,4.105141321818034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,64,128,1,fp8,fp8,0,1.1889920234680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,64,0,1,float16,float16,0,4.009504000345866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,64,128,1,float16,float16,0,0.9799359639485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,64,0,1,float16,fp8,0,4.023664156595866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,64,0,1,fp8,fp8,0,3.678677241007487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,64,128,1,float16,fp8,0,0.9107413291931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,64,0,1,float16,float16,0,4.124133427937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,64,128,1,fp8,fp8,0,1.2274719874064128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,64,128,1,float16,float16,0,0.5337706804275513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,64,0,1,fp8,fp8,0,3.623215993245443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,64,128,1,float16,fp8,0,0.5463946660359701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,64,0,1,float16,fp8,0,4.18070920308431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,64,128,1,fp8,fp8,0,0.6897226969401041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,64,128,1,float16,float16,0,0.505733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,64,0,1,float16,float16,0,2.067530632019043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,64,0,1,float16,float16,0,4.18393071492513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,64,0,1,fp8,fp8,0,3.6459840138753257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,64,128,1,float16,fp8,0,0.4851306676864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,64,0,1,float16,fp8,0,4.151365280151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,64,0,1,fp8,fp8,0,1.933637301127116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,64,0,1,float16,fp8,0,2.0918025970458984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,64,128,1,fp8,fp8,0,0.6663839817047119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,64,0,1,float16,float16,0,2.0616587003072104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,64,128,1,float16,float16,0,0.5073920090993246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,64,128,1,float16,fp8,0,0.5122933387756348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,64,128,1,fp8,fp8,0,0.6415786743164062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,64,0,1,fp8,fp8,0,1.8683199882507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,64,0,1,float16,fp8,0,2.0673653284708657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,64,128,1,float16,float16,0,0.5095040003458658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,64,0,1,float16,float16,0,2.0245866775512695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,64,128,1,float16,fp8,0,0.4901546637217204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,64,0,1,fp8,fp8,0,1.870682716369629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,64,128,1,fp8,fp8,0,0.6638186772664388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,64,0,1,float16,fp8,0,2.066095987955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,64,0,1,float16,float16,0,2.043445269266764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,64,128,1,float16,float16,0,0.4988853136698405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,64,128,1,float16,fp8,0,0.5099946657816569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,64,128,1,fp8,fp8,0,0.6631733179092407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,64,0,1,float16,fp8,0,2.0266772905985513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,64,0,1,fp8,fp8,0,1.9274932543436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,64,128,1,float16,float16,0,0.35715198516845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,64,0,1,float16,float16,0,2.040602684020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,64,128,1,float16,fp8,0,0.3538293441136678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,64,128,1,fp8,fp8,0,0.43490668137868244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,64,0,1,float16,float16,0,1.1575413544972737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,64,0,1,fp8,fp8,0,1.873088041941325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,64,0,1,float16,fp8,0,2.054821332295736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,64,0,1,float16,fp8,0,1.1417067050933838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,64,128,1,float16,float16,0,0.35250667730967206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,64,128,1,float16,fp8,0,0.35473068555196124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,64,128,1,fp8,fp8,0,0.42711468537648517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,64,0,1,fp8,fp8,0,1.0183040301005046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,64,128,1,float16,float16,0,0.3529546658198039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,64,0,1,float16,float16,0,1.161135991414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,64,128,1,float16,fp8,0,0.35176531473795575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,64,0,1,fp8,fp8,0,1.0114719867706299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,64,0,1,float16,fp8,0,1.1532320181528728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,64,128,1,fp8,fp8,0,0.431328018506368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,64,128,1,float16,float16,0,0.35151998202006024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,64,0,1,float16,float16,0,1.1601760387420654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,64,128,1,float16,fp8,0,0.3545386791229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,64,0,1,float16,fp8,0,1.1413919925689697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,64,128,1,fp8,fp8,0,0.4325919946034749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,64,0,1,fp8,fp8,0,1.011402686436971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,64,0,1,float16,float16,0,1.159552017847697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,64,128,1,float16,float16,0,0.35415999094645184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,64,0,1,float16,fp8,0,1.1585173606872559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,64,128,1,float16,fp8,0,0.35492265224456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,64,128,1,fp8,fp8,0,0.42978131771087646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,64,0,1,fp8,fp8,0,1.0119946797688801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,64,0,1,float16,float16,0,1.1425493558247883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,64,0,1,fp8,fp8,0,1.0205439726511638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,64,0,1,float16,fp8,0,1.159589370091756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,64,128,1,float16,float16,0,2.885786692301432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,64,128,1,float16,fp8,0,2.8407465616861978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,64,128,1,fp8,fp8,0,3.6501973470052085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,64,128,1,float16,float16,0,2.985706647237142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,64,128,1,float16,fp8,0,2.963717460632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,64,128,1,fp8,fp8,0,3.7623678843180337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,64,128,1,float16,float16,0,3.0010773340861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,64,0,1,fp8,fp8,0,9.46126937866211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,64,0,1,float16,float16,0,10.306074778238932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,64,0,1,float16,fp8,0,10.42136001586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,64,0,1,float16,float16,0,10.51312510172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,64,128,1,float16,fp8,0,2.9764480590820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,64,0,1,fp8,fp8,0,9.56667709350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,64,128,1,fp8,fp8,0,3.795109430948893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,64,0,1,float16,fp8,0,10.462581634521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,64,128,1,float16,float16,0,3.029221216837565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,64,128,1,float16,fp8,0,2.999733289082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,64,0,1,float16,float16,0,10.54806391398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,64,128,1,fp8,fp8,0,3.8514026006062827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,64,128,1,float16,float16,0,1.6745120684305828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,64,0,1,fp8,fp8,0,9.611370722452799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,64,128,1,float16,fp8,0,1.641434669494629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,64,0,1,float16,fp8,0,10.496559778849283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,64,128,1,fp8,fp8,0,2.1005493799845376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,64,128,1,float16,float16,0,1.3860534032185872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,64,0,1,float16,float16,0,5.601359685262044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,64,0,1,float16,float16,0,10.614277521769205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,64,128,1,float16,fp8,0,1.3976267178853352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,64,0,1,fp8,fp8,0,9.634538650512695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,64,0,1,float16,fp8,0,10.544970830281576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,64,0,1,fp8,fp8,0,5.041263898213704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,64,128,1,fp8,fp8,0,1.8312907218933105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,64,0,1,float16,fp8,0,5.624703725179036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,64,128,1,float16,float16,0,1.4528106053670247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,64,0,1,float16,float16,0,5.3709971110026045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,64,128,1,float16,fp8,0,1.4566720326741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,64,128,1,fp8,fp8,0,1.904597282409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,64,128,1,float16,float16,0,1.4944853782653809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,64,0,1,float16,fp8,0,5.154613176981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,64,0,1,fp8,fp8,0,4.831967989603679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,64,128,1,float16,fp8,0,1.4597759246826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,64,0,1,float16,float16,0,5.476117451985677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,64,128,1,fp8,fp8,0,1.9214186668395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,64,0,1,fp8,fp8,0,4.914922714233398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,64,128,1,float16,float16,0,1.504543940226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,64,0,1,float16,fp8,0,5.883034388224284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,64,0,1,float16,float16,0,5.335823694864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,64,128,1,float16,fp8,0,1.4960479736328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,64,0,1,fp8,fp8,0,4.846730550130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,64,128,1,fp8,fp8,0,1.930400053660075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,64,0,1,float16,fp8,0,5.714058558146159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,64,128,1,float16,float16,0,0.8210186958312988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,64,128,1,float16,fp8,0,0.7999200026194254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,64,128,1,fp8,fp8,0,1.1336373488108318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,64,0,1,float16,float16,0,5.285413424173991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,64,128,1,float16,float16,0,0.7397813002268473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,64,0,1,float16,float16,0,2.8111839294433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,64,0,1,float16,fp8,0,5.438816070556641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,64,0,1,float16,fp8,0,2.853856086730957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,64,0,1,fp8,fp8,0,4.871269226074219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,64,128,1,float16,fp8,0,0.7104640007019043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,64,0,1,fp8,fp8,0,2.496960004170736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,64,128,1,fp8,fp8,0,0.9409279823303223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,64,128,1,float16,float16,0,0.7238559722900391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,64,128,1,float16,fp8,0,0.7055413722991943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,64,128,1,fp8,fp8,0,0.9419146378835043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,64,0,1,float16,float16,0,2.6392107009887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,64,0,1,fp8,fp8,0,2.450613339742025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,64,0,1,float16,fp8,0,2.6046293576558432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,64,128,1,float16,float16,0,0.7268906434377035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,64,0,1,float16,float16,0,2.6136746406555176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,64,128,1,float16,fp8,0,0.7179733117421468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,64,0,1,fp8,fp8,0,2.3941866556803384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,64,0,1,float16,fp8,0,2.635109265645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,64,128,1,fp8,fp8,0,0.9496266841888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,64,128,1,float16,float16,0,0.741370677947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,64,0,1,float16,float16,0,2.6347786585489907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,64,128,1,float16,fp8,0,0.7229333718617758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,64,128,1,fp8,fp8,0,1.0098400115966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,64,128,1,float16,float16,0,0.45799465974171955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,64,0,1,float16,fp8,0,2.5854880015055337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,64,0,1,fp8,fp8,0,2.429525375366211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,64,128,1,float16,fp8,0,0.4530773162841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,64,0,1,float16,float16,0,2.702159881591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,64,128,1,fp8,fp8,0,0.5793333450953165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,64,0,1,float16,float16,0,1.385189374287923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,64,128,1,float16,float16,0,0.4097333351771037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,64,0,1,fp8,fp8,0,2.435365358988444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,64,0,1,float16,fp8,0,2.7003199259440103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,64,128,1,float16,fp8,0,0.3999093373616536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,64,0,1,fp8,fp8,0,1.3098399639129639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,64,0,1,float16,fp8,0,1.4804372787475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,64,128,1,fp8,fp8,0,0.5333066781361898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,64,128,1,float16,float16,0,0.42056532700856525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,64,0,1,float16,float16,0,1.400821367899577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,64,128,1,float16,fp8,0,0.39479466279347736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,64,128,1,fp8,fp8,0,0.5337546666463217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,64,0,1,fp8,fp8,0,1.2595252990722656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,64,0,1,float16,fp8,0,1.3809386889139812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,64,128,1,float16,float16,0,0.40962668259938556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,64,0,1,float16,float16,0,1.3801120122273762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,64,128,1,float16,fp8,0,0.40834665298461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,64,0,1,float16,fp8,0,1.3835253715515137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,64,0,1,fp8,fp8,0,1.313215970993042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,64,128,1,fp8,fp8,0,0.5188159942626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,64,128,1,float16,float16,0,0.4118880033493042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,64,0,1,float16,float16,0,1.3827733993530273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,64,128,1,float16,fp8,0,0.41034666697184247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,64,128,1,fp8,fp8,0,0.5284159978230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,64,0,1,float16,fp8,0,1.3881066640218098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,64,0,1,fp8,fp8,0,1.2914453347524006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,64,128,1,float16,float16,0,0.2889813383420308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,64,0,1,float16,float16,0,1.3931679725646973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,64,128,1,float16,fp8,0,0.28862933317820233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,64,128,1,fp8,fp8,0,0.3551040093104045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,64,0,1,float16,float16,0,0.7849493026733398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,64,0,1,float16,fp8,0,1.395407994588216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,64,0,1,fp8,fp8,0,1.288090705871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,64,128,1,float16,float16,0,0.28940266370773315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,64,0,1,float16,fp8,0,0.7857920328776041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,64,0,1,fp8,fp8,0,0.7007466952006022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,64,128,1,float16,fp8,0,0.2892906665802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,64,128,1,fp8,fp8,0,0.3505280017852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,64,0,1,float16,float16,0,0.7876373132069906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,64,128,1,float16,float16,0,0.28915733098983765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,64,128,1,float16,fp8,0,0.28913066784540814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,64,0,1,fp8,fp8,0,0.6949386596679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,64,128,1,fp8,fp8,0,0.3548266490300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,64,0,1,float16,fp8,0,0.7998720010121664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,64,0,1,float16,float16,0,0.7881226539611816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,64,0,1,float16,fp8,0,0.7869439919789633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,64,128,1,float16,float16,0,0.2882293264071147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,64,0,1,fp8,fp8,0,0.6949280103047689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,64,128,1,float16,fp8,0,0.28748265902201336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,64,0,1,float16,float16,0,0.7932799657185873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,64,128,1,fp8,fp8,0,0.3535360097885132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,64,128,1,float16,float16,0,0.2902453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,64,0,1,float16,fp8,0,0.7979733149210612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,64,128,1,float16,fp8,0,0.29017066955566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,64,0,1,fp8,fp8,0,0.7038453420003256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,64,128,1,fp8,fp8,0,0.3505653142929077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,64,0,1,float16,float16,0,0.787775993347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,64,0,1,float16,fp8,0,0.7934133211771647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,64,0,1,fp8,fp8,0,0.7006239891052246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,64,128,1,float16,float16,0,2.159279982248942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,64,128,1,float16,fp8,0,2.1378933588663735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,64,128,1,fp8,fp8,0,2.746783892313639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,64,128,1,float16,float16,0,2.2396000226338706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,64,128,1,float16,fp8,0,2.222650686899821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,64,0,1,float16,float16,0,6.215018590291341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,64,0,1,float16,fp8,0,6.302848180135091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,64,128,1,fp8,fp8,0,2.83244260152181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,64,0,1,fp8,fp8,0,5.681946436564128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,64,128,1,float16,float16,0,2.2525599797566733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,64,0,1,float16,float16,0,6.293968200683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,64,128,1,float16,fp8,0,2.2187466621398926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,64,128,1,fp8,fp8,0,2.830554644266764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,64,0,1,fp8,fp8,0,5.75001589457194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,64,128,1,float16,float16,0,2.2879467010498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,64,0,1,float16,fp8,0,6.2335465749104815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,64,128,1,float16,fp8,0,2.2422025998433432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,64,0,1,float16,float16,0,6.444400151570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,64,128,1,fp8,fp8,0,2.8766454060872397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,64,128,1,float16,float16,0,1.2348053455352783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,64,0,1,fp8,fp8,0,5.788927714029948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,64,0,1,float16,fp8,0,6.502480189005534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,64,128,1,float16,fp8,0,1.210213343302409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,64,0,1,float16,float16,0,6.347429275512695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,64,128,1,fp8,fp8,0,1.5685386657714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,64,128,1,float16,float16,0,1.0224800109863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,64,0,1,float16,float16,0,3.29146671295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,64,128,1,float16,fp8,0,1.0023199717203777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,64,0,1,fp8,fp8,0,5.834181467692058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,64,0,1,float16,fp8,0,6.2774613698323565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,64,0,1,fp8,fp8,0,3.065749486287435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,64,0,1,float16,fp8,0,3.384570757548014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,64,128,1,fp8,fp8,0,1.3391839663187664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,64,128,1,float16,float16,0,1.0495039621988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,64,0,1,float16,float16,0,3.062922795613607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,64,128,1,float16,fp8,0,1.036351998647054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,64,128,1,fp8,fp8,0,1.430415948232015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,64,0,1,float16,fp8,0,3.144048055013021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,64,0,1,fp8,fp8,0,2.8225440979003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,64,128,1,float16,float16,0,1.0649333000183105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,64,0,1,float16,float16,0,3.250394821166992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,64,128,1,float16,fp8,0,1.0672213236490886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,64,0,1,float16,fp8,0,3.131418546040853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,64,0,1,fp8,fp8,0,2.861152013142904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,64,128,1,fp8,fp8,0,1.4022986094156902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,64,128,1,float16,float16,0,1.1082506974538167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,64,0,1,float16,float16,0,3.364805221557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,64,128,1,float16,fp8,0,1.0681013266245525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,64,0,1,fp8,fp8,0,2.856517473856608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,64,128,1,fp8,fp8,0,1.4331307411193848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,64,128,1,float16,float16,0,0.6340800126393636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,64,0,1,float16,fp8,0,3.068431854248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,64,0,1,float16,float16,0,3.1157334645589194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,64,128,1,float16,fp8,0,0.6064213514328003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,64,128,1,fp8,fp8,0,0.8038453261057535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,64,0,1,float16,float16,0,1.6369279225667317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,64,128,1,float16,float16,0,0.5472000042597452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,64,0,1,float16,fp8,0,3.2293920516967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,64,0,1,fp8,fp8,0,2.910106658935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,64,0,1,float16,fp8,0,1.6346240043640137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,64,0,1,fp8,fp8,0,1.516757329305013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,64,128,1,float16,fp8,0,0.5470879872639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,64,128,1,fp8,fp8,0,0.726143995920817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,64,0,1,float16,float16,0,1.5643679300944011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,64,128,1,float16,float16,0,0.5420000155766805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,64,128,1,float16,fp8,0,0.5311520099639893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,64,0,1,float16,fp8,0,1.5626826286315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,64,0,1,fp8,fp8,0,1.471402645111084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,64,128,1,fp8,fp8,0,0.7286346753438314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,64,128,1,float16,float16,0,0.546671986579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,64,0,1,float16,float16,0,1.5731093088785808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,64,128,1,float16,fp8,0,0.5442933241526285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,64,128,1,fp8,fp8,0,0.7199040253957113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,64,0,1,float16,fp8,0,1.5943039258321126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,64,0,1,fp8,fp8,0,1.4478507041931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,64,128,1,float16,float16,0,0.5638720194498698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,64,0,1,float16,float16,0,1.6149120330810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,64,128,1,float16,fp8,0,0.5526560147603353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,64,0,1,float16,fp8,0,1.5735626220703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,64,128,1,fp8,fp8,0,0.7438186804453532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,64,0,1,fp8,fp8,0,1.4512267112731934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,64,128,1,float16,float16,0,0.33819735050201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,64,128,1,float16,fp8,0,0.3349119822184245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,64,128,1,fp8,fp8,0,0.43933868408203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,64,0,1,float16,float16,0,1.5869760513305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,64,0,1,float16,float16,0,0.8543093204498291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,64,0,1,float16,fp8,0,1.6183679898579915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,64,0,1,fp8,fp8,0,1.4934132893880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,64,128,1,float16,float16,0,0.3131680091222127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,64,128,1,float16,fp8,0,0.30187733968098956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,64,0,1,float16,fp8,0,0.8519306977589926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,64,0,1,fp8,fp8,0,0.7773706912994385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,64,128,1,fp8,fp8,0,0.4010293483734131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,64,0,1,float16,float16,0,0.8470719655354818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,64,128,1,float16,float16,0,0.3116426666577657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,64,0,1,float16,fp8,0,0.840394655863444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,64,0,1,fp8,fp8,0,0.752229372660319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,64,128,1,float16,fp8,0,0.2997866670290629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,64,128,1,fp8,fp8,0,0.3988906542460124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,64,0,1,float16,float16,0,0.8473546504974365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,64,0,1,float16,fp8,0,0.8358986377716064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,64,128,1,float16,float16,0,0.3135040005048116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,64,0,1,fp8,fp8,0,0.7529546419779459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,64,128,1,float16,fp8,0,0.3084106643994649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,64,128,1,fp8,fp8,0,0.4010293483734131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,64,0,1,float16,float16,0,0.849829355875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,64,128,1,float16,float16,0,0.31673065821329754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,64,128,1,float16,fp8,0,0.3102666735649109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,64,0,1,fp8,fp8,0,0.7553173700968424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,64,0,1,float16,fp8,0,0.8465387026468912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,64,128,1,fp8,fp8,0,0.40398399035135907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,64,0,1,float16,float16,0,0.8467413584391276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,64,128,1,float16,float16,0,0.22314133246739706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,64,128,1,float16,fp8,0,0.22467732429504395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,64,0,1,float16,fp8,0,0.8495199680328369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,64,128,1,fp8,fp8,0,0.27317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,64,0,1,float16,float16,0,0.4710506598154704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,64,0,1,fp8,fp8,0,0.7591679890950521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,64,0,1,float16,fp8,0,0.4751573403676351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,64,128,1,float16,float16,0,0.2217173377672831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,64,0,1,fp8,fp8,0,0.43796801567077637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,64,128,1,float16,fp8,0,0.22217067082722983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,64,0,1,float16,float16,0,0.47257598241170246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,64,128,1,fp8,fp8,0,0.27245867252349854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,64,128,1,float16,float16,0,0.22292800744374594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,64,0,1,float16,fp8,0,0.4715679883956909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,64,0,1,float16,float16,0,0.46720532576243085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,64,128,1,float16,fp8,0,0.22142932812372842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,64,0,1,fp8,fp8,0,0.44150400161743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,64,128,1,fp8,fp8,0,0.27398399511973065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,64,0,1,float16,fp8,0,0.46779199441274005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,64,128,1,float16,float16,0,0.2230666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,64,128,1,float16,fp8,0,0.22179200251897177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,64,0,1,fp8,fp8,0,0.4442506631215413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,64,0,1,float16,float16,0,0.4671306610107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,64,128,1,fp8,fp8,0,0.273525337378184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,64,0,1,float16,fp8,0,0.47396798928578693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,64,128,1,float16,float16,0,0.22326932350794473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,64,0,1,fp8,fp8,0,0.43802134195963544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,64,128,1,float16,fp8,0,0.22460265954335532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,64,0,1,float16,float16,0,0.4662719964981079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,64,128,1,fp8,fp8,0,0.27238933245340985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,64,0,1,float16,fp8,0,0.47117865085601807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,64,0,1,fp8,fp8,0,0.44436268011728924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,64,128,1,float16,float16,0,2.837653477986654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,64,128,1,float16,fp8,0,2.8197971979777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,64,128,1,float16,float16,0,2.97434139251709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,64,128,1,fp8,fp8,0,3.591909408569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,64,0,1,float16,float16,0,6.083386739095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,64,0,1,fp8,fp8,0,5.616101582845052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,64,128,1,float16,fp8,0,2.932117462158203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,64,0,1,float16,fp8,0,6.0932051340738935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,64,128,1,fp8,fp8,0,3.7434027989705405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,64,128,1,float16,float16,0,2.9689067204793296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,64,0,1,float16,float16,0,6.371039708455403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,64,128,1,float16,fp8,0,2.9465227127075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,64,0,1,float16,fp8,0,6.256746927897136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,64,128,1,fp8,fp8,0,3.772197405497233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,64,0,1,fp8,fp8,0,5.762218475341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,64,0,1,float16,float16,0,6.4105650583903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,64,128,1,float16,float16,0,3.028986612955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,64,128,1,float16,fp8,0,2.994112014770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,64,0,1,float16,fp8,0,6.3762772878011065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,64,128,1,fp8,fp8,0,3.810341199239095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,64,128,1,float16,float16,0,1.6702507336934407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,64,0,1,fp8,fp8,0,5.772266387939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,64,0,1,float16,float16,0,6.277306874593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,64,128,1,float16,fp8,0,1.6261067390441895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,64,128,1,fp8,fp8,0,2.0819039344787598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,64,0,1,float16,float16,0,3.397989273071289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,64,128,1,float16,float16,0,1.359557310740153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,64,0,1,float16,fp8,0,6.394351959228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,64,0,1,float16,fp8,0,3.4192479451497397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,64,0,1,fp8,fp8,0,3.0900001525878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,64,128,1,float16,fp8,0,1.3286666870117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,64,0,1,fp8,fp8,0,5.840005238850911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,64,128,1,fp8,fp8,0,1.8439092636108398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,64,0,1,float16,float16,0,3.0923945109049478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,64,128,1,float16,float16,0,1.4862987200419109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,64,128,1,float16,fp8,0,1.4267999331156414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,64,0,1,float16,fp8,0,3.003413200378418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,64,128,1,fp8,fp8,0,1.8740906715393066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,64,0,1,fp8,fp8,0,2.8100268046061196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,64,128,1,float16,float16,0,1.4640320142110188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,64,0,1,float16,float16,0,3.1712640126546225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,64,128,1,float16,fp8,0,1.4814559618632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,64,0,1,fp8,fp8,0,2.893498738606771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,64,0,1,float16,fp8,0,3.115872065226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,64,128,1,fp8,fp8,0,1.884101390838623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,64,128,1,float16,float16,0,1.5075680414835613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,64,0,1,float16,float16,0,3.107237180074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,64,128,1,float16,fp8,0,1.4778399467468262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,64,0,1,float16,fp8,0,3.082218805948893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,64,0,1,fp8,fp8,0,2.9061174392700195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,64,128,1,fp8,fp8,0,1.9246187210083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,64,128,1,float16,float16,0,0.8178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,64,128,1,float16,fp8,0,0.793503999710083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,64,0,1,float16,float16,0,3.128490765889486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,64,0,1,float16,fp8,0,3.231413205464681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,64,0,1,float16,float16,0,1.6583040555318196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,64,128,1,fp8,fp8,0,1.0335573355356853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,64,128,1,float16,float16,0,0.6839679876963297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,64,0,1,fp8,fp8,0,2.9399998982747397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,64,0,1,float16,fp8,0,1.615045388539632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,64,0,1,fp8,fp8,0,1.569375991821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,64,128,1,float16,fp8,0,0.6937759717305502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,64,128,1,fp8,fp8,0,0.9034667015075684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,64,0,1,float16,float16,0,1.5491894086201985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,64,128,1,float16,float16,0,0.7206239700317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,64,128,1,float16,fp8,0,0.7004000345865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,64,0,1,float16,fp8,0,1.5038080215454102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,64,0,1,fp8,fp8,0,1.4086079597473145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,64,128,1,fp8,fp8,0,0.9115413029988607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,64,0,1,float16,float16,0,1.5273226102193196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,64,128,1,float16,float16,0,0.7164053122202555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,64,128,1,float16,fp8,0,0.6913440227508545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,64,0,1,float16,fp8,0,1.5408587455749512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,64,0,1,fp8,fp8,0,1.4491626421610515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,64,128,1,fp8,fp8,0,0.9193600018819174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,64,0,1,float16,float16,0,1.5262826283772786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,64,128,1,float16,float16,0,0.7129226525624593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,64,0,1,float16,fp8,0,1.5213546752929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,64,128,1,float16,fp8,0,0.7066986560821533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,64,0,1,fp8,fp8,0,1.4271999994913738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,64,128,1,fp8,fp8,0,0.9529973665873209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,64,128,1,float16,float16,0,0.4211626847585042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,64,0,1,float16,float16,0,1.5665119489034016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,64,0,1,float16,fp8,0,1.5323306719462078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,64,128,1,float16,fp8,0,0.4118880033493042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,64,0,1,float16,float16,0,0.8376533190409342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,64,128,1,fp8,fp8,0,0.5430239836374918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,64,0,1,fp8,fp8,0,1.4326400756835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,64,128,1,float16,float16,0,0.39683731396993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,64,0,1,float16,fp8,0,0.831658681233724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,64,0,1,fp8,fp8,0,0.7882026831309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,64,128,1,float16,fp8,0,0.3655093510945638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,64,0,1,float16,float16,0,0.7908373673756918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,64,128,1,fp8,fp8,0,0.4935733477274577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,64,128,1,float16,float16,0,0.3879679838816325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,64,0,1,float16,fp8,0,0.789743979771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,64,0,1,fp8,fp8,0,0.7327626546223959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,64,128,1,float16,fp8,0,0.36750932534535724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,64,0,1,float16,float16,0,0.7951359748840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,64,128,1,fp8,fp8,0,0.5007146596908569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,64,128,1,float16,float16,0,0.38096535205841064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,64,0,1,float16,fp8,0,0.7861440181732178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,64,0,1,fp8,fp8,0,0.7566826343536377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,64,128,1,float16,fp8,0,0.3674079974492391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,64,0,1,float16,float16,0,0.7979839642842611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,64,128,1,fp8,fp8,0,0.4914826552073161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,64,128,1,float16,float16,0,0.3859626849492391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,64,0,1,float16,fp8,0,0.7921653588612875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,64,128,1,float16,fp8,0,0.38169066111246747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,64,0,1,fp8,fp8,0,0.7640319665273031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,64,128,1,fp8,fp8,0,0.4930453300476074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,64,0,1,float16,float16,0,0.809770663579305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,64,0,1,float16,fp8,0,0.7994453112284342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,64,128,1,float16,float16,0,0.23181867599487305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,64,128,1,float16,fp8,0,0.2344800035158793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,64,0,1,fp8,fp8,0,0.7511839866638184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,64,0,1,float16,float16,0,0.4546826680501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,64,128,1,fp8,fp8,0,0.3076853354771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,64,0,1,float16,fp8,0,0.4447733163833618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,64,128,1,float16,float16,0,0.2113386591275533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,64,0,1,fp8,fp8,0,0.41176533699035645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,64,128,1,float16,fp8,0,0.2116159995396932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,64,0,1,float16,float16,0,0.43268799781799316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,64,128,1,fp8,fp8,0,0.27561066548029584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,64,0,1,float16,fp8,0,0.4329013427098592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,64,0,1,fp8,fp8,0,0.38572267691294354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,64,128,1,float16,float16,0,0.20908800760904947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,64,128,1,float16,fp8,0,0.21187732617060342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,64,0,1,float16,float16,0,0.4307733376820882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,64,128,1,fp8,fp8,0,0.27849066257476807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,64,0,1,float16,fp8,0,0.43245331446329754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,64,128,1,float16,float16,0,0.21621867020924887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,64,0,1,fp8,fp8,0,0.3842613299687703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,64,128,1,float16,fp8,0,0.2129759987195333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,64,128,1,float16,float16,0,0.21659733851750693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,64,0,1,float16,float16,0,0.43325865268707275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,64,128,1,fp8,fp8,0,0.28037865956624347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,64,0,1,float16,fp8,0,0.4298666715621948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,64,0,1,fp8,fp8,0,0.38788266976674396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,64,128,1,float16,fp8,0,0.21357866128285727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,64,0,1,float16,float16,0,0.43509864807128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,64,128,1,fp8,fp8,0,0.2839786609013875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,64,128,1,float16,float16,0,0.1567306617895762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,64,0,1,float16,fp8,0,0.4352320035298665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,64,0,1,fp8,fp8,0,0.38978131612141925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,64,0,1,float16,float16,0,0.2550506591796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,64,128,1,float16,fp8,0,0.15819733341534933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,64,128,1,fp8,fp8,0,0.19498666127522787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,64,0,1,float16,fp8,0,0.25568000475565594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,64,128,1,float16,float16,0,0.155157337586085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,64,0,1,fp8,fp8,0,0.24092799425125122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,64,0,1,float16,float16,0,0.2526400089263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,64,128,1,float16,fp8,0,0.15522666772206625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,64,0,1,float16,float16,0,0.25305600961049396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,64,128,1,fp8,fp8,0,0.19207467635472616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,64,0,1,float16,fp8,0,0.2536533276240031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,64,0,1,fp8,fp8,0,0.23928000529607138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,64,128,1,float16,float16,0,0.15596800049146017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,64,128,1,float16,fp8,0,0.156058669090271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,64,128,1,fp8,fp8,0,0.19377066691716513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,64,0,1,float16,fp8,0,0.2529386679331462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,64,0,1,fp8,fp8,0,0.23922133445739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,64,128,1,float16,float16,0,0.15613333384195963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,64,0,1,float16,float16,0,0.25225067138671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,64,128,1,float16,fp8,0,0.1553439994653066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,64,128,1,fp8,fp8,0,0.19335466623306274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,64,0,1,float16,fp8,0,0.25335999329884845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,64,0,1,float16,float16,0,0.25380265712738037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,64,128,1,fp8,fp8,0,0.19317332903544107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,64,0,1,fp8,fp8,0,0.23963733514149985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,64,0,1,float16,fp8,0,0.25512532393137616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,64,128,1,float16,float16,0,0.15614933768908182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,64,128,1,float16,fp8,0,0.15546666582425436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,64,0,1,fp8,fp8,0,0.2396106719970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,64,128,1,float16,float16,0,2.068864027659098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,64,128,1,float16,fp8,0,2.0551199913024902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,64,128,1,fp8,fp8,0,2.67411200205485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,64,128,1,float16,float16,0,2.1360960006713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,64,0,1,float16,float16,0,3.816399892171224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,64,0,1,fp8,fp8,0,3.4895200729370117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,64,0,1,float16,fp8,0,3.722304026285807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,64,128,1,float16,fp8,0,2.1185332934061685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,64,128,1,fp8,fp8,0,2.723775863647461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,64,0,1,float16,float16,0,3.871429443359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,64,128,1,float16,float16,0,2.1553279558817544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,64,128,1,float16,fp8,0,2.136021296183268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,64,0,1,float16,fp8,0,3.8070828119913735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,64,0,1,fp8,fp8,0,3.5336532592773438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,64,0,1,float16,float16,0,3.830592155456543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,64,128,1,fp8,fp8,0,2.7438131968180337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,64,128,1,float16,float16,0,2.197754700978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,64,128,1,float16,fp8,0,2.182485262552897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,64,0,1,fp8,fp8,0,3.5797119140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,64,0,1,float16,fp8,0,3.8379573822021484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,64,128,1,fp8,fp8,0,2.815631866455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,64,128,1,float16,float16,0,1.2271040280659993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,64,0,1,float16,float16,0,3.987039883931478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,64,128,1,float16,fp8,0,1.2028266588846843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,64,128,1,fp8,fp8,0,1.5587093035380046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,64,0,1,float16,fp8,0,3.933098793029785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,64,0,1,float16,float16,0,2.100560029347738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,64,128,1,float16,float16,0,1.0281439622243245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,64,0,1,fp8,fp8,0,3.595674514770508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,64,0,1,float16,fp8,0,2.035205364227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,64,0,1,fp8,fp8,0,1.954202651977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,64,128,1,float16,fp8,0,0.9901440143585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,64,128,1,fp8,fp8,0,1.3226293722788494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,64,0,1,float16,float16,0,1.8830506006876628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,64,128,1,float16,float16,0,1.0221707026163738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,64,0,1,float16,fp8,0,1.8699092864990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,64,128,1,float16,fp8,0,1.0109439690907795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,64,0,1,float16,float16,0,1.8611092567443848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,64,0,1,fp8,fp8,0,1.7285653750101726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,64,128,1,fp8,fp8,0,1.3498613039652507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,64,0,1,float16,fp8,0,1.8561973571777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,64,128,1,float16,float16,0,1.0291306972503662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,64,0,1,fp8,fp8,0,1.7892053922017415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,64,128,1,float16,fp8,0,1.0212106704711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,64,128,1,fp8,fp8,0,1.3624480565388997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,64,0,1,float16,float16,0,1.883962631225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,64,128,1,float16,float16,0,1.0623679955800374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,64,0,1,float16,fp8,0,1.8671092987060547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,64,128,1,float16,fp8,0,1.0595626831054688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,64,0,1,fp8,fp8,0,1.771183967590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,64,128,1,float16,float16,0,0.6086506843566895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,64,0,1,float16,float16,0,1.9047306378682454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,64,128,1,fp8,fp8,0,1.3904053370157878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,64,0,1,float16,float16,0,1.0337440172831218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,64,128,1,float16,fp8,0,0.6026453177134196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,64,128,1,fp8,fp8,0,0.7598773638407389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,64,0,1,float16,fp8,0,1.8889706929524739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,64,0,1,fp8,fp8,0,1.8037333488464355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,64,0,1,float16,fp8,0,1.0201120376586914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,64,0,1,fp8,fp8,0,0.9589866797129313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,64,128,1,float16,float16,0,0.5177280108133951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,64,0,1,float16,float16,0,0.9687626361846924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,64,128,1,float16,fp8,0,0.5172213315963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,64,128,1,fp8,fp8,0,0.6834666728973389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,64,128,1,float16,float16,0,0.5256533225377401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,64,0,1,float16,fp8,0,0.9251840114593506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,64,0,1,fp8,fp8,0,0.884768009185791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,64,128,1,float16,fp8,0,0.5184746583302816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,64,0,1,float16,float16,0,0.9453919728597006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,64,128,1,fp8,fp8,0,0.7046773433685303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,64,128,1,float16,float16,0,0.529749313990275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,64,0,1,float16,fp8,0,0.946885347366333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,64,128,1,float16,fp8,0,0.5220959981282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,64,0,1,fp8,fp8,0,0.8880639870961508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,64,0,1,float16,float16,0,0.9767200152079264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,64,128,1,fp8,fp8,0,0.6933386325836182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,64,128,1,float16,float16,0,0.5370506842931112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,64,0,1,float16,fp8,0,0.9447999795277914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,64,0,1,fp8,fp8,0,0.8971786499023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,64,128,1,float16,fp8,0,0.5305333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,64,0,1,float16,float16,0,0.9652000268300375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,64,128,1,float16,float16,0,0.3243839939435323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,64,128,1,fp8,fp8,0,0.7001813252766927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,64,0,1,float16,fp8,0,0.9772799809773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,64,128,1,float16,fp8,0,0.31701866785685223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,64,0,1,fp8,fp8,0,0.9035039742787679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,64,0,1,float16,float16,0,0.5332479874293009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,64,128,1,fp8,fp8,0,0.4087626536687215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,64,128,1,float16,fp8,0,0.28295467297236127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,64,0,1,float16,fp8,0,0.5423893531163534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,64,128,1,float16,float16,0,0.284661332766215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,64,0,1,float16,fp8,0,0.5044106642405192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,64,0,1,fp8,fp8,0,0.4614666700363159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,64,0,1,fp8,fp8,0,0.5002613464991251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,64,0,1,float16,float16,0,0.5031573375066122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,64,128,1,fp8,fp8,0,0.36788801352183026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,64,128,1,fp8,fp8,0,0.3801120122273763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,64,128,1,float16,float16,0,0.2904053330421448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,64,0,1,float16,float16,0,0.5056373278299967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,64,128,1,float16,fp8,0,0.28524800141652423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,64,0,1,float16,fp8,0,0.5013920068740845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,64,0,1,float16,float16,0,0.5062719980875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,64,0,1,fp8,fp8,0,0.4570560057957967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,64,128,1,float16,float16,0,0.29283199707667035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,64,128,1,float16,fp8,0,0.2850506703058879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,64,128,1,fp8,fp8,0,0.3779199918111165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,64,0,1,float16,fp8,0,0.5040320158004761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,64,0,1,fp8,fp8,0,0.4606506824493408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,64,128,1,float16,float16,0,0.2906399965286255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,64,0,1,float16,float16,0,0.5102293491363525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,64,128,1,float16,fp8,0,0.29283734162648517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,64,128,1,fp8,fp8,0,0.38209064801534015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,64,128,1,float16,float16,0,0.1783733367919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,64,0,1,float16,fp8,0,0.5083306630452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,64,0,1,float16,fp8,0,0.28227200110753375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,64,0,1,fp8,fp8,0,0.46454401810963947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,64,0,1,float16,float16,0,0.28043200572331745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,64,128,1,float16,fp8,0,0.17917867501576742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,64,128,1,fp8,fp8,0,0.2369813323020935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,64,0,1,fp8,fp8,0,0.26678399244944256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,64,128,1,float16,float16,0,0.16314666469891867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,64,0,1,float16,float16,0,0.2624160051345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,64,128,1,float16,fp8,0,0.1628106633822123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,64,128,1,fp8,fp8,0,0.21597866217295328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,64,0,1,float16,fp8,0,0.25991467634836835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,64,0,1,fp8,fp8,0,0.25098133087158203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,64,128,1,float16,float16,0,0.1641759971777598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,64,0,1,float16,float16,0,0.26185067494710285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,64,128,1,float16,fp8,0,0.16260266304016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,64,128,1,fp8,fp8,0,0.21694932381312051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,64,0,1,float16,fp8,0,0.2604479988416036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,64,128,1,float16,fp8,0,0.16363733013470969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,64,0,1,fp8,fp8,0,0.24942932526270548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,64,128,1,float16,float16,0,0.16545599699020386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,64,0,1,float16,float16,0,0.2632426619529724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,64,128,1,fp8,fp8,0,0.2189813256263733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,64,0,1,float16,fp8,0,0.2625066637992859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,64,0,1,fp8,fp8,0,0.2505279978116353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,64,128,1,float16,float16,0,0.16690667470296225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,64,0,1,float16,float16,0,0.2646186749140422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,64,128,1,float16,fp8,0,0.16632533073425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,64,128,1,fp8,fp8,0,0.22092799345652261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,64,128,1,float16,fp8,0,0.12498133381207784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,64,0,1,float16,fp8,0,0.26443199316660565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,64,0,1,fp8,fp8,0,0.2539253234863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,64,128,1,float16,float16,0,0.12472533186276753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,64,0,1,float16,float16,0,0.17220799128214517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,64,128,1,fp8,fp8,0,0.15542399883270264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,64,0,1,float16,fp8,0,0.172325332959493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,64,0,1,fp8,fp8,0,0.1630346675713857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,64,128,1,float16,float16,0,0.12129066387812297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,64,0,1,float16,float16,0,0.16839466492335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,64,128,1,float16,fp8,0,0.12054399649302165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,64,128,1,fp8,fp8,0,0.14338666200637817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,64,128,1,float16,fp8,0,0.12106666962305705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,64,0,1,float16,fp8,0,0.1690559983253479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,64,0,1,fp8,fp8,0,0.16055466731389365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,64,128,1,float16,float16,0,0.12083199620246887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,64,0,1,float16,float16,0,0.1702666680018107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,64,128,1,fp8,fp8,0,0.14873600006103516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,64,0,1,float16,fp8,0,0.16768532991409302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,64,0,1,fp8,fp8,0,0.16081600387891135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,64,0,1,float16,fp8,0,0.1692319909731547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,64,0,1,fp8,fp8,0,0.162282665570577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,64,128,1,float16,float16,0,0.12174933155377705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,64,0,1,float16,float16,0,0.1693013310432434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,64,128,1,float16,fp8,0,0.12223999698956807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,64,128,1,fp8,fp8,0,0.14545067151387533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,64,128,1,float16,float16,0,0.12311999996503194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,64,0,1,float16,float16,0,0.16969066858291626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,64,128,1,float16,fp8,0,0.12284266948699951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,64,128,1,fp8,fp8,0,0.15281599760055542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,64,0,1,float16,fp8,0,0.16993065675099692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,64,0,1,fp8,fp8,0,0.16245866815249124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,64,128,1,float16,float16,0,2.7674773534139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,64,128,1,float16,fp8,0,2.7302560806274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,64,128,1,float16,float16,0,2.769866625467936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,64,0,1,float16,float16,0,3.97327454884847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,64,128,1,fp8,fp8,0,3.451989491780599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,64,0,1,fp8,fp8,0,3.6192906697591147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,64,0,1,float16,fp8,0,3.948762575785319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,64,0,1,float16,float16,0,4.019205411275228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,64,128,1,float16,fp8,0,2.7432800928751626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,64,128,1,fp8,fp8,0,3.5722506841023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,64,128,1,float16,float16,0,2.8011252085367837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,64,0,1,fp8,fp8,0,3.6780052185058594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,64,0,1,float16,fp8,0,3.953866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,64,128,1,float16,fp8,0,2.760832150777181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,64,0,1,float16,float16,0,4.0147145589192705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,64,128,1,fp8,fp8,0,3.5559040705362954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,64,128,1,float16,float16,0,2.855536142985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,64,0,1,fp8,fp8,0,3.7093334197998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,64,0,1,float16,fp8,0,3.9682718912760415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,64,128,1,float16,fp8,0,2.8169066111246743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,64,0,1,float16,float16,0,4.19374942779541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,64,128,1,fp8,fp8,0,3.6174186070760093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,64,128,1,float16,float16,0,1.6104373931884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,64,128,1,float16,fp8,0,1.577679951985677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,64,0,1,float16,fp8,0,4.1426239013671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,64,0,1,float16,float16,0,2.2615307172139487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,64,128,1,fp8,fp8,0,2.0739466349283853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,64,128,1,float16,float16,0,1.3391040166219075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,64,0,1,fp8,fp8,0,3.731818517049154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,64,0,1,float16,fp8,0,2.227183977762858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,64,0,1,fp8,fp8,0,2.1013439496358237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,64,128,1,float16,fp8,0,1.3232693672180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,64,0,1,float16,float16,0,1.9458826382954915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,64,128,1,fp8,fp8,0,1.7390400568644206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,64,128,1,float16,float16,0,1.3518932660420735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,64,0,1,fp8,fp8,0,1.8149545987447102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,64,0,1,float16,fp8,0,1.931322733561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,64,128,1,float16,fp8,0,1.3467787106831868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,64,0,1,float16,float16,0,1.9768266677856445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,64,128,1,fp8,fp8,0,1.768298625946045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,64,0,1,float16,fp8,0,1.9940427144368489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,64,0,1,fp8,fp8,0,1.8363146781921387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,64,128,1,float16,float16,0,1.3689546585083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,64,128,1,float16,fp8,0,1.3699359893798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,64,0,1,float16,float16,0,1.9916586875915527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,64,128,1,fp8,fp8,0,1.7545706431070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,64,128,1,float16,float16,0,1.4367733001708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,64,0,1,fp8,fp8,0,1.8520533243815105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,64,0,1,float16,fp8,0,1.979599952697754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,64,128,1,float16,fp8,0,1.385994593302409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,64,0,1,float16,float16,0,2.0262559254964194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,64,128,1,float16,float16,0,0.8101812998453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,64,128,1,fp8,fp8,0,1.8035945892333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,64,128,1,float16,fp8,0,0.788586695988973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,64,0,1,float16,float16,0,1.1162772973378499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,64,128,1,fp8,fp8,0,1.0015146732330322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,64,0,1,fp8,fp8,0,1.855514685312907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,64,0,1,float16,fp8,0,2.008783976236979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,64,0,1,float16,fp8,0,1.0924266974131267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,64,128,1,float16,float16,0,0.672976016998291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,64,0,1,fp8,fp8,0,1.0485599835713704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,64,128,1,float16,fp8,0,0.6626453399658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,64,0,1,float16,float16,0,0.9782613118489584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,64,128,1,fp8,fp8,0,0.8666400114695231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,64,0,1,float16,fp8,0,0.9591946601867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,64,128,1,float16,float16,0,0.6786133448282877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,64,0,1,fp8,fp8,0,0.9071626663208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,64,128,1,float16,fp8,0,0.6725333531697592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,64,0,1,float16,float16,0,0.9791093667348226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,64,128,1,fp8,fp8,0,0.8892426490783691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,64,128,1,float16,float16,0,0.6751999855041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,64,0,1,fp8,fp8,0,0.9097973505655924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,64,0,1,float16,fp8,0,0.9766560395558676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,64,128,1,float16,fp8,0,0.6704160372416178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,64,0,1,float16,float16,0,0.9901493390401205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,64,128,1,fp8,fp8,0,0.8948853015899658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,64,0,1,float16,fp8,0,0.975536028544108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,64,0,1,fp8,fp8,0,0.928213357925415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,64,128,1,float16,float16,0,0.6860640048980713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,64,128,1,float16,fp8,0,0.6867360273996989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,64,0,1,float16,float16,0,0.9879946708679199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,64,128,1,fp8,fp8,0,0.906218687693278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,64,128,1,float16,float16,0,0.41012799739837646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,64,128,1,float16,fp8,0,0.39946667353312176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,64,0,1,float16,float16,0,0.5651839971542358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,64,0,1,float16,fp8,0,0.9920852979024252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,64,0,1,fp8,fp8,0,0.9428479671478271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,64,128,1,fp8,fp8,0,0.5158559878667196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,64,0,1,float16,fp8,0,0.5538666645685831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,64,128,1,float16,float16,0,0.35229333241780597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,64,0,1,fp8,fp8,0,0.5259040196736654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,64,128,1,float16,fp8,0,0.345909317334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,64,0,1,float16,float16,0,0.5039573510487875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,64,128,1,fp8,fp8,0,0.46541865666707355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,64,0,1,float16,fp8,0,0.5059413512547811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,64,128,1,float16,float16,0,0.35354665915171307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,64,0,1,fp8,fp8,0,0.4790773391723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,64,128,1,float16,fp8,0,0.35131200154622394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,64,0,1,float16,float16,0,0.5108000040054321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,64,128,1,fp8,fp8,0,0.4660693407058716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,64,0,1,float16,fp8,0,0.5138826767603556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,64,0,1,fp8,fp8,0,0.4810719887415568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,64,128,1,float16,float16,0,0.3672959804534912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,64,0,1,float16,float16,0,0.5085386832555135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,64,128,1,float16,fp8,0,0.3558080196380615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,64,128,1,fp8,fp8,0,0.4697813193003337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,64,0,1,float16,float16,0,0.5209066470464071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,64,0,1,float16,fp8,0,0.5104906558990479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,64,0,1,fp8,fp8,0,0.4806453386942546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,64,128,1,float16,float16,0,0.3686986764272054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,64,128,1,float16,fp8,0,0.36715734004974365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,64,128,1,fp8,fp8,0,0.4718773365020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,64,0,1,float16,fp8,0,0.5177813371022543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,64,128,1,float16,float16,0,0.22034666935602823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,64,0,1,fp8,fp8,0,0.48950934410095215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,64,0,1,float16,float16,0,0.30482133229573566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,64,128,1,float16,fp8,0,0.22009066740671793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,64,128,1,fp8,fp8,0,0.28913066784540814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,64,0,1,float16,fp8,0,0.29451199372609455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,64,0,1,fp8,fp8,0,0.27558932701746625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,64,128,1,float16,float16,0,0.19435199101765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,64,128,1,float16,fp8,0,0.1948266625404358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,64,0,1,float16,float16,0,0.2752693295478821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,64,128,1,fp8,fp8,0,0.2627786596616109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,64,0,1,float16,fp8,0,0.2748639980951945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,64,128,1,float16,fp8,0,0.19555199146270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,64,0,1,float16,float16,0,0.2773173252741496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,64,0,1,fp8,fp8,0,0.25462400913238525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,64,128,1,float16,float16,0,0.19827200969060263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,64,128,1,fp8,fp8,0,0.25938665866851807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,64,0,1,float16,fp8,0,0.27802666028340656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,64,128,1,float16,float16,0,0.2006346583366394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,64,0,1,fp8,fp8,0,0.2537173430124919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,64,0,1,float16,float16,0,0.2804373304049174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,64,128,1,float16,fp8,0,0.19821866353352866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,64,128,1,fp8,fp8,0,0.26527466376622516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,64,0,1,float16,fp8,0,0.2747466762860616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,64,0,1,fp8,fp8,0,0.25598933299382526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,64,128,1,float16,float16,0,0.20145066579182944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,64,0,1,float16,float16,0,0.2831786672274272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,64,128,1,float16,fp8,0,0.1997013290723165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,64,128,1,fp8,fp8,0,0.2670240004857381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,64,0,1,float16,fp8,0,0.2842453320821126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,64,0,1,fp8,fp8,0,0.2570026715596517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,64,128,1,float16,float16,0,0.12706133723258972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,64,0,1,float16,float16,0,0.1620746652285258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,64,128,1,float16,fp8,0,0.1293226679166158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,64,128,1,fp8,fp8,0,0.16731733083724976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,64,0,1,float16,fp8,0,0.16321600476900736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,64,0,1,fp8,fp8,0,0.155541330575943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,64,128,1,float16,float16,0,0.11441600322723389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,64,0,1,float16,float16,0,0.14778133233388266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,64,128,1,float16,fp8,0,0.11596266428629558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,64,128,1,fp8,fp8,0,0.1479039986928304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,64,0,1,float16,fp8,0,0.14727999766667685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,64,0,1,fp8,fp8,0,0.141594668229421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,64,128,1,float16,float16,0,0.11628266175587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,64,0,1,float16,float16,0,0.14710932970046997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,64,128,1,float16,fp8,0,0.116976002852122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,64,128,1,fp8,fp8,0,0.15416533748308817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,64,0,1,float16,fp8,0,0.1477013329664866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,64,0,1,fp8,fp8,0,0.14128533005714417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,64,128,1,float16,float16,0,0.11641066273053487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,64,0,1,float16,float16,0,0.14948800206184387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,64,128,1,float16,fp8,0,0.11577066779136658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,64,0,1,float16,float16,0,0.15061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,64,128,1,fp8,fp8,0,0.15716266632080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,64,0,1,float16,fp8,0,0.14858133594195047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,64,0,1,fp8,fp8,0,0.14297067125638327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,64,0,1,fp8,fp8,0,0.14646933476130167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,64,128,1,float16,float16,0,0.1178986628850301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,64,128,1,float16,fp8,0,0.11843732992808025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,64,128,1,fp8,fp8,0,0.15706666310628256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,64,0,1,float16,fp8,0,0.14913066228230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,64,128,1,float16,float16,0,0.0918880005677541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,64,0,1,float16,float16,0,0.10456533233324687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,64,128,1,float16,fp8,0,0.09115200241406758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,64,128,1,fp8,fp8,0,0.11526399850845337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,64,0,1,float16,fp8,0,0.10437333583831787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,64,0,1,fp8,fp8,0,0.10044266780217488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,64,128,1,float16,float16,0,0.08700799942016602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,64,0,1,float16,float16,0,0.10126933455467224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,64,128,1,float16,fp8,0,0.08747200171152751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,64,128,1,fp8,fp8,0,0.10379733641942342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,64,0,1,float16,fp8,0,0.10148266951243083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,64,0,1,fp8,fp8,0,0.0981066624323527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,64,128,1,float16,float16,0,0.0869599978129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,64,0,1,float16,float16,0,0.10162666440010071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,64,128,1,float16,fp8,0,0.08727467060089111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,64,128,1,fp8,fp8,0,0.10269332925478618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,64,0,1,float16,fp8,0,0.10172800223032634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,64,0,1,fp8,fp8,0,0.09827733039855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,64,128,1,float16,float16,0,0.0869760016600291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,64,0,1,float16,float16,0,0.10159466663996379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,64,128,1,float16,fp8,0,0.08701866865158081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,64,128,1,fp8,fp8,0,0.10299733281135559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,64,128,1,float16,fp8,0,0.08699199557304382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,64,0,1,float16,fp8,0,0.10124799609184265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,64,0,1,fp8,fp8,0,0.09787733356157939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,64,128,1,float16,float16,0,0.08742933471997578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,64,0,1,float16,float16,0,0.10178666313489278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,64,128,1,fp8,fp8,0,0.10439999898274739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,64,0,1,float16,fp8,0,0.10181867082913716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,64,0,1,fp8,fp8,0,0.09805867075920105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,64,128,1,float16,float16,0,2.0432000160217285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,64,128,1,float16,fp8,0,2.015434741973877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,64,0,1,float16,float16,0,2.5984479586283364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,64,128,1,float16,float16,0,2.03385066986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,64,128,1,fp8,fp8,0,2.565978686014811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,64,0,1,fp8,fp8,0,2.347205320994059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,64,0,1,float16,fp8,0,2.5888266563415527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,64,0,1,float16,float16,0,2.5921546618143716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,64,128,1,float16,fp8,0,2.004821300506592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,64,128,1,float16,float16,0,2.0509066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,64,128,1,fp8,fp8,0,2.5997865994771323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,64,0,1,fp8,fp8,0,2.3858559926350913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,64,0,1,float16,fp8,0,2.561493396759033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,64,128,1,float16,fp8,0,2.020005385080973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,64,0,1,float16,float16,0,2.616741339365641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,64,128,1,fp8,fp8,0,2.587578614552816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,64,128,1,float16,float16,0,2.1122399965922036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,64,0,1,fp8,fp8,0,2.391786734263102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,64,128,1,float16,fp8,0,2.068511962890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,64,0,1,float16,fp8,0,2.582320054372152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,64,0,1,float16,float16,0,2.666261355082194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,64,128,1,fp8,fp8,0,2.66320530573527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,64,128,1,float16,float16,0,1.2028906345367432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,64,128,1,float16,fp8,0,1.1696159839630127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,64,0,1,float16,float16,0,1.4831679662068684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,64,0,1,float16,fp8,0,2.682037353515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,64,128,1,fp8,fp8,0,1.491066614786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,64,0,1,float16,fp8,0,1.4488959312438965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,64,0,1,fp8,fp8,0,2.4595786730448403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,64,128,1,float16,float16,0,0.9833333492279053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,64,0,1,fp8,fp8,0,1.3817226092020671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,64,128,1,float16,fp8,0,0.9993493556976318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,64,0,1,float16,float16,0,1.2625706990559895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,64,128,1,fp8,fp8,0,1.2966880003611247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,64,0,1,float16,fp8,0,1.2495466868082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,64,0,1,fp8,fp8,0,1.194970687230428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,64,128,1,float16,float16,0,1.0086346467336018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,64,128,1,float16,fp8,0,0.9880320231119791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,64,0,1,float16,float16,0,1.2757493654886882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,64,0,1,float16,fp8,0,1.2685600121815999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,64,128,1,fp8,fp8,0,1.330954631169637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,64,128,1,float16,float16,0,1.0103519757588704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,64,0,1,fp8,fp8,0,1.1962186495463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,64,128,1,float16,fp8,0,1.0036853154500325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,64,0,1,float16,float16,0,1.3128266334533691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,64,128,1,fp8,fp8,0,1.304912010828654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,64,0,1,float16,fp8,0,1.2763306299845378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,64,0,1,fp8,fp8,0,1.2261066436767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,64,128,1,float16,float16,0,1.0474239985148113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,64,128,1,float16,fp8,0,1.0337920188903809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,64,0,1,float16,float16,0,1.3180853525797527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,64,128,1,fp8,fp8,0,1.3422026634216309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,64,128,1,float16,float16,0,0.6017546653747559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,64,0,1,float16,fp8,0,1.3150506814320881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,64,0,1,fp8,fp8,0,1.2196533679962158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,64,128,1,float16,fp8,0,0.5911360184351603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,64,0,1,float16,float16,0,0.7556640307108561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,64,128,1,fp8,fp8,0,0.7509333292643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,64,0,1,float16,fp8,0,0.7278239727020264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,64,128,1,float16,float16,0,0.5020586649576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,64,0,1,fp8,fp8,0,0.6857759952545166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,64,128,1,float16,fp8,0,0.495525320370992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,64,0,1,float16,float16,0,0.6513973474502563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,64,128,1,fp8,fp8,0,0.6649706761042277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,64,0,1,fp8,fp8,0,0.6043999989827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,64,0,1,float16,fp8,0,0.645034670829773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,64,128,1,float16,float16,0,0.5077173312505087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,64,0,1,float16,float16,0,0.6467146476109823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,64,128,1,float16,fp8,0,0.5043306748072306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,64,128,1,fp8,fp8,0,0.6637599865595499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,64,0,1,float16,fp8,0,0.6491413513819376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,64,0,1,fp8,fp8,0,0.609226663907369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,64,128,1,float16,float16,0,0.5163733164469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,64,128,1,float16,fp8,0,0.5051786502202352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,64,0,1,float16,float16,0,0.659663995107015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,64,128,1,fp8,fp8,0,0.6694613297780355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,64,0,1,float16,fp8,0,0.6511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,64,0,1,fp8,fp8,0,0.6073013146718343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,64,128,1,float16,float16,0,0.5242506663004557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,64,0,1,float16,float16,0,0.6671573321024576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,64,128,1,float16,fp8,0,0.5190186500549316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,64,128,1,fp8,fp8,0,0.6856853167215983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,64,0,1,float16,fp8,0,0.6619946559270223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,64,0,1,fp8,fp8,0,0.6186293363571167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,64,128,1,float16,float16,0,0.3131306568781535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,64,0,1,float16,fp8,0,0.3795253435770671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,64,0,1,float16,float16,0,0.38334933916727704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,64,128,1,float16,fp8,0,0.3080693284670512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,64,128,1,fp8,fp8,0,0.3935146729151408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,64,0,1,fp8,fp8,0,0.35310932000478107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,64,128,1,float16,float16,0,0.2719200054804484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,64,0,1,float16,float16,0,0.3396906852722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,64,128,1,float16,fp8,0,0.26606933275858563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,64,128,1,fp8,fp8,0,0.35684800148010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,64,0,1,float16,fp8,0,0.337285319964091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,64,0,1,fp8,fp8,0,0.3130506674448649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,64,128,1,float16,float16,0,0.2777600089708964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,64,0,1,float16,float16,0,0.3440213203430176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,64,128,1,float16,fp8,0,0.2703146735827128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,64,128,1,fp8,fp8,0,0.35467199484507245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,64,0,1,float16,fp8,0,0.3423839807510376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,64,0,1,fp8,fp8,0,0.31564799944559735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,64,128,1,float16,float16,0,0.27381332715352374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,64,0,1,float16,float16,0,0.34863468011220294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,64,128,1,float16,fp8,0,0.27562665939331055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,64,128,1,fp8,fp8,0,0.3664586544036865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,64,0,1,float16,fp8,0,0.3454666535059611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,64,0,1,fp8,fp8,0,0.3153013388315837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,64,128,1,float16,float16,0,0.2779200077056885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,64,0,1,float16,float16,0,0.3498186667760213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,64,128,1,float16,fp8,0,0.2779200077056885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,64,128,1,fp8,fp8,0,0.3702720006306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,64,0,1,float16,fp8,0,0.35301868120829266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,64,128,1,float16,float16,0,0.16947199900945029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,64,0,1,fp8,fp8,0,0.32316799958546955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,64,0,1,float16,float16,0,0.21887467304865518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,64,128,1,float16,fp8,0,0.16962667306264242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,64,128,1,float16,float16,0,0.15001066525777182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,64,128,1,fp8,fp8,0,0.22140266497929892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,64,0,1,float16,fp8,0,0.21855467557907104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,64,0,1,fp8,fp8,0,0.1933599909146627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,64,0,1,float16,float16,0,0.17922665675481161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,64,128,1,float16,fp8,0,0.15054399768511453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,64,128,1,fp8,fp8,0,0.20122132698694864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,64,0,1,float16,fp8,0,0.17471466461817423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,64,0,1,fp8,fp8,0,0.17174933354059854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,64,128,1,float16,float16,0,0.15400532881418863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,64,0,1,float16,float16,0,0.18140800793965658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,64,128,1,float16,fp8,0,0.15014400084813437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,64,128,1,fp8,fp8,0,0.2046346664428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,64,0,1,float16,fp8,0,0.1763253410657247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,64,0,1,fp8,fp8,0,0.17108267545700073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,64,128,1,float16,float16,0,0.15346133708953857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,64,0,1,float16,float16,0,0.18266133467356363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,64,128,1,float16,fp8,0,0.15171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,64,128,1,fp8,fp8,0,0.20427733659744263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,64,0,1,float16,fp8,0,0.18050134181976318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,64,0,1,fp8,fp8,0,0.17325333754221597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,64,128,1,fp8,fp8,0,0.20837332804997763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,64,128,1,float16,float16,0,0.15481066703796387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,64,0,1,float16,float16,0,0.18331199884414673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,64,128,1,float16,float16,0,0.10010133186976115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,64,128,1,float16,fp8,0,0.10133866469065349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,64,128,1,float16,fp8,0,0.15667733550071716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,64,0,1,float16,fp8,0,0.18148799737294516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,64,0,1,fp8,fp8,0,0.10967466235160828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,64,0,1,fp8,fp8,0,0.1776319940884908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,64,0,1,float16,float16,0,0.11431466539700826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,64,128,1,fp8,fp8,0,0.13397866487503052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,64,0,1,float16,fp8,0,0.11364799737930298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,64,128,1,float16,float16,0,0.09008533755938213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,64,0,1,float16,float16,0,0.10215999682744344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,64,128,1,float16,fp8,0,0.09060266613960266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,64,128,1,fp8,fp8,0,0.11372266213099162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,64,128,1,float16,fp8,0,0.09036800265312195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,64,0,1,float16,fp8,0,0.10256533821423848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,64,0,1,fp8,fp8,0,0.09700266520182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,64,128,1,float16,float16,0,0.09149332841237386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,64,0,1,float16,float16,0,0.10398399829864502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,64,128,1,fp8,fp8,0,0.11512533823649089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,64,0,1,float16,fp8,0,0.10246933499972026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,64,0,1,fp8,fp8,0,0.09905067086219788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,64,128,1,float16,float16,0,0.09206933776537578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,64,0,1,float16,float16,0,0.10367467006047566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,64,128,1,float16,fp8,0,0.09066133697827657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,64,128,1,fp8,fp8,0,0.11840533216794331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,64,0,1,float16,fp8,0,0.10380267103513081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,64,0,1,fp8,fp8,0,0.09884267052014668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,64,128,1,float16,float16,0,0.09357866644859314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,64,0,1,float16,float16,0,0.1050933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,64,128,1,float16,fp8,0,0.0932426651318868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,64,128,1,fp8,fp8,0,0.12345066666603088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,64,0,1,float16,fp8,0,0.1050879955291748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,64,0,1,fp8,fp8,0,0.10174399614334106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,64,128,1,float16,float16,0,0.07239466905593872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,64,0,1,float16,float16,0,0.07580266892910004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,64,128,1,float16,fp8,0,0.07120533287525177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,64,128,1,fp8,fp8,0,0.09337066610654195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,64,0,1,float16,fp8,0,0.07646933197975159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,64,0,1,fp8,fp8,0,0.07517866790294647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,64,128,1,float16,float16,0,0.06906666855017345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,64,0,1,float16,float16,0,0.07513600091139476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,64,128,1,float16,fp8,0,0.06974933544794719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,64,128,1,fp8,fp8,0,0.0836906631787618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,64,0,1,float16,fp8,0,0.0753119985262553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,64,0,1,fp8,fp8,0,0.07225066423416138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,64,128,1,float16,float16,0,0.0701279987891515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,64,0,1,float16,float16,0,0.07513066629568736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,64,128,1,float16,fp8,0,0.06923200190067291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,64,128,1,fp8,fp8,0,0.084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,64,0,1,float16,fp8,0,0.07578133543332417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,64,0,1,fp8,fp8,0,0.07215466598669688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,64,128,1,float16,float16,0,0.06945066650708516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,64,0,1,float16,float16,0,0.07448533177375793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,64,128,1,float16,fp8,0,0.0689333329598109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,64,128,1,fp8,fp8,0,0.08592533071835835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,64,0,1,float16,fp8,0,0.07521066566308339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,64,0,1,fp8,fp8,0,0.07245866457621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,64,0,1,fp8,fp8,0,0.07310933371384938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,64,128,1,float16,float16,0,0.06982933481534322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,64,0,1,float16,float16,0,0.07437333464622498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,64,128,1,float16,fp8,0,0.06926933427651723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,64,128,1,fp8,fp8,0,0.08610133330027263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,64,0,1,float16,fp8,0,0.07477866609891255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,64,128,1,float16,float16,0,2.351909319559733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,64,128,1,float16,fp8,0,2.3380533854166665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,64,0,1,float16,float16,0,2.6746241251627603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,64,0,1,float16,fp8,0,2.6820319493611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,64,128,1,fp8,fp8,0,3.165477434794108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,64,0,1,fp8,fp8,0,2.466906706492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,64,128,1,float16,float16,0,2.3962507247924805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,64,128,1,float16,fp8,0,2.376335938771566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,64,0,1,float16,float16,0,2.723994572957357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,64,0,1,float16,fp8,0,2.6928745905558267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,64,128,1,fp8,fp8,0,3.174367904663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,64,0,1,fp8,fp8,0,2.501189390818278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,64,128,1,float16,float16,0,2.398085276285807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,64,0,1,float16,float16,0,2.7379414240519204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,64,128,1,float16,fp8,0,2.384005387624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,64,128,1,fp8,fp8,0,3.2112372716267905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,64,0,1,fp8,fp8,0,2.522794723510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,64,0,1,float16,fp8,0,2.706501324971517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,64,128,1,float16,float16,0,2.4416586558024087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,64,128,1,float16,fp8,0,2.455519994099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,64,0,1,float16,float16,0,2.7626399993896484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,64,128,1,float16,float16,0,1.491322676340739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,64,128,1,float16,fp8,0,1.4479786554972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,64,0,1,float16,float16,0,1.672378698984782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,64,128,1,fp8,fp8,0,3.3049920399983725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,64,0,1,float16,fp8,0,2.7656052907307944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,64,0,1,float16,fp8,0,1.6131092707316081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,64,128,1,fp8,fp8,0,1.7992587089538574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,64,0,1,fp8,fp8,0,2.615034739176432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,64,128,1,float16,float16,0,1.1771306991577148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,64,0,1,fp8,fp8,0,1.448618729909261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,64,128,1,float16,fp8,0,1.1790453592936199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,64,0,1,float16,float16,0,1.3622080485026042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,64,0,1,fp8,fp8,0,1.251845359802246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,64,128,1,fp8,fp8,0,1.6041280428568523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,64,0,1,float16,fp8,0,1.3407200177510579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,64,128,1,float16,float16,0,1.2144532998402913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,64,0,1,float16,float16,0,1.375973383585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,64,128,1,float16,fp8,0,1.2105600039164226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,64,0,1,float16,fp8,0,1.3776000340779622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,64,128,1,fp8,fp8,0,1.5892906188964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,64,0,1,fp8,fp8,0,1.2659839789072673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,64,128,1,float16,float16,0,1.2220853169759114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,64,0,1,float16,float16,0,1.3835147221883137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,64,128,1,float16,fp8,0,1.2249066829681396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,64,0,1,float16,fp8,0,1.369658629099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,64,0,1,fp8,fp8,0,1.2874080340067546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,64,128,1,fp8,fp8,0,1.6374133427937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,64,128,1,float16,float16,0,1.2123146851857503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,64,128,1,float16,fp8,0,1.216426690419515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,64,0,1,float16,float16,0,1.4018294016520183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,64,0,1,float16,fp8,0,1.387386639912923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,64,128,1,float16,float16,0,0.7584053675333658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,64,128,1,fp8,fp8,0,1.6666080156962078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,64,0,1,float16,float16,0,0.838485320409139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,64,128,1,float16,fp8,0,0.7272960344950358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,64,0,1,fp8,fp8,0,1.2926506996154785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,64,128,1,fp8,fp8,0,0.905029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,64,0,1,float16,fp8,0,0.8123146692911783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,64,128,1,float16,float16,0,0.613477349281311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,64,0,1,fp8,fp8,0,0.7274293104807535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,64,0,1,float16,float16,0,0.6977600256601969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,64,128,1,float16,fp8,0,0.6076000134150187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,64,128,1,fp8,fp8,0,0.80021866162618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,64,0,1,float16,fp8,0,0.6982933680216471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,64,0,1,fp8,fp8,0,0.6275466680526733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,64,128,1,float16,float16,0,0.6137386560440063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,64,0,1,float16,float16,0,0.7033440272013346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,64,128,1,float16,fp8,0,0.6206613381703695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,64,128,1,float16,float16,0,0.6238133509953817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,64,0,1,float16,fp8,0,0.696394681930542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,64,128,1,fp8,fp8,0,0.8260693550109863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,64,0,1,fp8,fp8,0,0.6234133243560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,64,128,1,float16,fp8,0,0.6171253522237142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,64,0,1,float16,float16,0,0.6983093420664469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,64,128,1,fp8,fp8,0,0.825546662012736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,64,0,1,float16,fp8,0,0.7100906372070312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,64,0,1,fp8,fp8,0,0.6392853260040283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,64,128,1,float16,float16,0,0.6319146553675333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,64,128,1,float16,fp8,0,0.6212746699651083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,64,0,1,float16,float16,0,0.7068640391031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,64,128,1,float16,float16,0,0.3884853521982829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,64,0,1,fp8,fp8,0,0.660698652267456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,64,0,1,float16,fp8,0,0.7030453681945801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,64,128,1,fp8,fp8,0,0.8367146650950114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,64,0,1,float16,float16,0,0.4304533402125041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,64,128,1,float16,fp8,0,0.378762682278951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,64,128,1,fp8,fp8,0,0.46960000197092694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,64,0,1,float16,fp8,0,0.420085350672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,64,128,1,float16,float16,0,0.32124267021814984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,64,0,1,fp8,fp8,0,0.37432531515757245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,64,0,1,float16,float16,0,0.3590346574783325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,64,128,1,float16,fp8,0,0.3184906641642253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,64,128,1,fp8,fp8,0,0.42631999651590985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,64,0,1,float16,fp8,0,0.3617973327636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,64,0,1,fp8,fp8,0,0.3282613356908162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,64,128,1,fp8,fp8,0,0.4245920181274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,64,128,1,float16,float16,0,0.31998932361602783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,64,0,1,float16,float16,0,0.3651893138885498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,64,128,1,float16,fp8,0,0.3221493363380432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,64,0,1,float16,fp8,0,0.36235201358795166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,64,0,1,fp8,fp8,0,0.3312480052312215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,64,128,1,float16,float16,0,0.325162669022878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,64,0,1,float16,float16,0,0.36408531665802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,64,128,1,float16,fp8,0,0.323034663995107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,64,0,1,float16,fp8,0,0.36717867851257324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,64,128,1,fp8,fp8,0,0.4283999999364217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,64,0,1,fp8,fp8,0,0.33159999052683514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,64,128,1,float16,float16,0,0.32650667428970337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,64,128,1,fp8,fp8,0,0.42906665802001953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,64,128,1,float16,float16,0,0.20649067560831705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,64,0,1,float16,float16,0,0.3702239990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,64,128,1,float16,fp8,0,0.32542399565378827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,64,0,1,float16,float16,0,0.22714666525522867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,64,0,1,float16,fp8,0,0.37119468053181964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,64,0,1,fp8,fp8,0,0.3363413413365682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,64,128,1,float16,fp8,0,0.2023306687672933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,64,128,1,fp8,fp8,0,0.2516319950421651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,64,0,1,float16,fp8,0,0.22342934211095175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,64,128,1,float16,fp8,0,0.1731040080388387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,64,0,1,fp8,fp8,0,0.1934773325920105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,64,128,1,float16,float16,0,0.17176532745361328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,64,0,1,float16,float16,0,0.1949066718419393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,64,128,1,float16,float16,0,0.17388266324996948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,64,128,1,fp8,fp8,0,0.2323360045750936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,64,0,1,float16,fp8,0,0.19454934199651083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,64,0,1,fp8,fp8,0,0.1732106606165568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,64,0,1,float16,float16,0,0.19531200329462686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,64,128,1,float16,float16,0,0.17461333672205606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,64,128,1,float16,fp8,0,0.17306133111317953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,64,128,1,fp8,fp8,0,0.23094399770100912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,64,0,1,float16,fp8,0,0.19705599546432495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,64,0,1,fp8,fp8,0,0.1717066764831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,64,0,1,float16,float16,0,0.1968053380648295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,64,128,1,float16,fp8,0,0.17522666851679483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,64,128,1,float16,float16,0,0.17909334103266397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,64,128,1,fp8,fp8,0,0.23250667254130045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,64,0,1,float16,fp8,0,0.19618133703867593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,64,0,1,fp8,fp8,0,0.1745013395945231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,64,0,1,float16,float16,0,0.19960532585779825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,64,128,1,float16,fp8,0,0.1784906586011251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,64,0,1,float16,fp8,0,0.20043732722600302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,64,0,1,float16,float16,0,0.12178666392962138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,64,128,1,fp8,fp8,0,0.23520533243815103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,64,128,1,fp8,fp8,0,0.1450506647427877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,64,0,1,float16,fp8,0,0.11970667044321696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,64,0,1,fp8,fp8,0,0.17450666427612305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,64,128,1,float16,float16,0,0.11460266510645549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,64,128,1,float16,fp8,0,0.11451733112335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,64,0,1,fp8,fp8,0,0.10685333609580994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,64,128,1,float16,float16,0,0.09742933511734009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,64,0,1,float16,fp8,0,0.10337066650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,64,0,1,float16,float16,0,0.10423466563224792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,64,128,1,float16,fp8,0,0.09866133332252502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,64,128,1,fp8,fp8,0,0.12983999649683634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,64,0,1,fp8,fp8,0,0.09486400087674458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,64,128,1,float16,float16,0,0.09850666920344035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,64,0,1,float16,float16,0,0.10393066207567851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,64,128,1,float16,fp8,0,0.09842666983604431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,64,128,1,fp8,fp8,0,0.13476799925168356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,64,0,1,float16,fp8,0,0.10544000069300334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,64,0,1,fp8,fp8,0,0.09591999650001526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,64,128,1,float16,float16,0,0.09974400202433269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,64,0,1,float16,float16,0,0.10496000448862712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,64,128,1,float16,fp8,0,0.10012800494829814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,64,128,1,fp8,fp8,0,0.1330933372179667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,64,0,1,float16,fp8,0,0.10590400298436482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,64,0,1,fp8,fp8,0,0.09679999947547913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,64,128,1,float16,float16,0,0.1011946698029836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,64,128,1,float16,float16,0,0.0678773323694865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,64,0,1,float16,float16,0,0.10692266623179118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,64,128,1,float16,fp8,0,0.10188800096511841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,64,128,1,fp8,fp8,0,0.1362559994061788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,64,0,1,float16,fp8,0,0.10689066847165425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,64,0,1,fp8,fp8,0,0.09757333000500996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,64,0,1,float16,float16,0,0.06698133548100789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,64,128,1,float16,fp8,0,0.06695466736952464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,64,128,1,fp8,fp8,0,0.09096533060073853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,64,0,1,float16,fp8,0,0.06635199983914693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,64,0,1,fp8,fp8,0,0.06396799782911937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,64,128,1,float16,float16,0,0.058592001597086586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,64,0,1,float16,float16,0,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,64,128,1,float16,fp8,0,0.0588266650835673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,64,128,1,fp8,fp8,0,0.07552533348401387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,64,0,1,float16,fp8,0,0.060640002290407814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,64,0,1,fp8,fp8,0,0.05585066477457682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,64,128,1,float16,float16,0,0.059343998630841575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,64,0,1,float16,float16,0,0.061018665631612144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,64,128,1,float16,fp8,0,0.059232001503308616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,64,128,1,fp8,fp8,0,0.07496533294518788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,64,128,1,fp8,fp8,0,0.07600533465544383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,64,0,1,float16,fp8,0,0.06047466893990835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,64,0,1,fp8,fp8,0,0.05663999915122986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,64,128,1,float16,float16,0,0.060080001751581825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,64,0,1,float16,float16,0,0.062122667829195656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,64,128,1,float16,fp8,0,0.05975466469923655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,64,0,1,float16,fp8,0,0.06112533311049143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,64,0,1,fp8,fp8,0,0.05605866511662801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,64,128,1,float16,float16,0,0.06109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,64,0,1,float16,float16,0,0.061648001273473106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,64,128,1,float16,fp8,0,0.061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,64,128,1,fp8,fp8,0,0.08032533526420593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,64,0,1,float16,fp8,0,0.06247999767462412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,64,0,1,fp8,fp8,0,0.05685866872469584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,64,128,1,float16,float16,0,0.04242133100827535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,64,0,1,float16,float16,0,0.039994666973749794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,64,128,1,float16,fp8,0,0.04124800115823746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,64,128,1,fp8,fp8,0,0.05253866811593374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,64,128,1,float16,fp8,0,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,64,0,1,float16,fp8,0,0.0410453329483668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,64,0,1,fp8,fp8,0,0.03641066700220108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,64,0,1,float16,fp8,0,0.03881600002447764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,64,128,1,float16,float16,0,0.039503999054431915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,64,0,1,float16,float16,0,0.037674665451049805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,64,128,1,fp8,fp8,0,0.0499893327554067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,64,0,1,fp8,fp8,0,0.03600533306598663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,64,128,1,float16,float16,0,0.03887466589609782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,64,0,1,float16,float16,0,0.03792533278465271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,64,128,1,float16,fp8,0,0.03977599988381068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,64,0,1,float16,float16,0,0.03817066550254822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,64,128,1,fp8,fp8,0,0.050714666644732155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,64,0,1,float16,fp8,0,0.03786666691303253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,64,0,1,fp8,fp8,0,0.03559466699759165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,64,0,1,fp8,fp8,0,0.0374293327331543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,64,128,1,float16,float16,0,0.040378667414188385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,64,128,1,float16,fp8,0,0.04075733323891958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,64,128,1,fp8,fp8,0,0.051914667089780174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,64,0,1,float16,fp8,0,0.03904533386230469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,64,128,1,float16,float16,0,0.04147200038035711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,64,0,1,float16,float16,0,0.038933334251244865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,64,128,1,float16,fp8,0,0.04088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,64,128,1,fp8,fp8,0,0.051632001996040344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,64,0,1,float16,fp8,0,0.039493332306543984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,64,0,1,fp8,fp8,0,0.03712533414363861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,64,128,1,float16,float16,0,2.2170559565226235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,64,0,1,float16,float16,0,2.2036800384521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,64,128,1,float16,fp8,0,2.2176267306009927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,64,0,1,fp8,fp8,0,1.985962708791097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,64,0,1,float16,fp8,0,2.208090623219808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,64,128,1,float16,float16,0,2.249215920766195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,64,128,1,fp8,fp8,0,2.961728096008301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,64,128,1,float16,fp8,0,2.2431252797444663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,64,0,1,float16,float16,0,2.2220800717671714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,64,0,1,fp8,fp8,0,2.032309373219808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,64,0,1,float16,fp8,0,2.227680047353109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,64,128,1,float16,float16,0,2.2668533325195312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,64,0,1,float16,float16,0,2.2525866826375327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,64,128,1,fp8,fp8,0,2.9958346684773765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,64,128,1,float16,fp8,0,2.2834879557291665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,64,0,1,fp8,fp8,0,2.052192052205404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,64,0,1,float16,fp8,0,2.24180269241333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,64,128,1,float16,float16,0,2.308896064758301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,64,128,1,fp8,fp8,0,3.0156160990397134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,64,0,1,float16,float16,0,2.2912373542785645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,64,128,1,float16,fp8,0,2.3086400032043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,64,0,1,float16,fp8,0,2.3400266965230307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,64,128,1,fp8,fp8,0,3.046799977620443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,64,128,1,float16,float16,0,1.4395947456359863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,64,0,1,float16,float16,0,1.4126399358113606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,64,128,1,float16,fp8,0,1.3729173342386882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,64,0,1,fp8,fp8,0,2.102656046549479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,64,0,1,float16,fp8,0,1.3658933639526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,64,128,1,fp8,fp8,0,1.6782293319702148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,64,0,1,fp8,fp8,0,1.1954987049102783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,64,128,1,float16,float16,0,1.0907413164774578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,64,0,1,float16,float16,0,1.1177706718444824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,64,128,1,float16,fp8,0,1.114144007364909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,64,128,1,fp8,fp8,0,1.4605120023091633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,64,0,1,fp8,fp8,0,0.9812533060709635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,64,0,1,float16,fp8,0,1.1194453239440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,64,128,1,float16,float16,0,1.1233546733856201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,64,0,1,float16,float16,0,1.1317226886749268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,64,128,1,float16,fp8,0,1.1139307022094727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,64,0,1,float16,fp8,0,1.1119360129038494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,64,128,1,fp8,fp8,0,1.4982986450195312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,64,0,1,fp8,fp8,0,0.9840853214263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,64,128,1,float16,float16,0,1.1302026907602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,64,0,1,float16,float16,0,1.139514684677124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,64,128,1,float16,fp8,0,1.1285920143127441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,64,0,1,float16,fp8,0,1.1056000391642253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,64,128,1,fp8,fp8,0,1.5294186274210613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,64,0,1,fp8,fp8,0,1.0200693607330322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,64,128,1,float16,float16,0,1.1551520029703777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,64,0,1,float16,float16,0,1.1241599718729656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,64,128,1,float16,fp8,0,1.1405920187632244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,64,0,1,float16,fp8,0,1.127130667368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,64,128,1,float16,float16,0,0.7227413654327393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,64,128,1,fp8,fp8,0,1.5482133229573567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,64,0,1,fp8,fp8,0,1.0320320129394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,64,0,1,float16,float16,0,0.7099626859029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,64,128,1,float16,fp8,0,0.6954879760742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,64,128,1,fp8,fp8,0,0.8500479857126871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,64,0,1,float16,fp8,0,0.6862826347351074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,64,128,1,float16,float16,0,0.5734026829401652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,64,128,1,float16,fp8,0,0.5641280015309652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,64,0,1,float16,fp8,0,0.5609813531239828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,64,0,1,fp8,fp8,0,0.6010400056838989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,64,0,1,float16,float16,0,0.5631626844406128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,64,128,1,fp8,fp8,0,0.7577226956685384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,64,0,1,fp8,fp8,0,0.49483199914296466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,64,128,1,float16,float16,0,0.5740586519241333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,64,128,1,float16,fp8,0,0.5720106760660807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,64,0,1,float16,float16,0,0.5705013275146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,64,128,1,fp8,fp8,0,0.7611733277638754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,64,0,1,float16,fp8,0,0.5641546646753947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,64,0,1,fp8,fp8,0,0.5162026484807333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,64,128,1,float16,float16,0,0.5734080076217651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,64,0,1,float16,float16,0,0.5724266767501831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,64,128,1,float16,fp8,0,0.5768053531646729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,64,0,1,fp8,fp8,0,0.5108906825383505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,64,0,1,float16,fp8,0,0.5732213258743286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,64,128,1,fp8,fp8,0,0.770032008488973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,64,128,1,float16,float16,0,0.6003626585006714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,64,0,1,float16,float16,0,0.5774986743927002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,64,128,1,float16,fp8,0,0.5900746583938599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,64,128,1,fp8,fp8,0,0.7756960391998291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,64,0,1,float16,fp8,0,0.5753973325093588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,64,0,1,fp8,fp8,0,0.528218666712443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,64,128,1,float16,float16,0,0.37142399946848553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,64,0,1,float16,float16,0,0.36429333686828613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,64,128,1,float16,fp8,0,0.3588426510492961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,64,0,1,float16,fp8,0,0.3500906626383464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,64,128,1,fp8,fp8,0,0.4434666633605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,64,128,1,float16,fp8,0,0.30642666419347125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,64,0,1,fp8,fp8,0,0.3088480035463969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,64,128,1,float16,float16,0,0.3003893295923869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,64,0,1,float16,float16,0,0.29461334149042767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,64,128,1,fp8,fp8,0,0.39671464761098224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,64,0,1,float16,fp8,0,0.29107733567555744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,64,0,1,fp8,fp8,0,0.2639840046564738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,64,128,1,float16,float16,0,0.3070080081621806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,64,0,1,float16,float16,0,0.2956906755765279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,64,128,1,float16,fp8,0,0.304149329662323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,64,0,1,float16,fp8,0,0.29654399553934735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,64,128,1,fp8,fp8,0,0.40238932768503827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,64,0,1,fp8,fp8,0,0.26425600051879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,64,128,1,float16,float16,0,0.3057279984156291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,64,0,1,float16,float16,0,0.30058133602142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,64,128,1,float16,fp8,0,0.30526934067408246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,64,128,1,fp8,fp8,0,0.40084266662597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,64,0,1,float16,fp8,0,0.299130658308665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,64,0,1,fp8,fp8,0,0.2682560086250305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,64,128,1,float16,fp8,0,0.3124213417371114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,64,128,1,float16,float16,0,0.3102826674779256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,64,0,1,float16,float16,0,0.30404800176620483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,64,128,1,fp8,fp8,0,0.40597331523895264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,64,0,1,float16,fp8,0,0.3022613326708476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,64,0,1,fp8,fp8,0,0.27033599217732746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,64,128,1,float16,float16,0,0.19813867410024008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,64,0,1,float16,fp8,0,0.18804800510406494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,64,0,1,float16,float16,0,0.19220266739527384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,64,128,1,float16,float16,0,0.16337066888809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,64,0,1,float16,float16,0,0.16010666886965433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,64,128,1,float16,fp8,0,0.19343467553456625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,64,128,1,fp8,fp8,0,0.23854400714238486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,64,0,1,fp8,fp8,0,0.16268799702326456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,64,128,1,float16,fp8,0,0.1646666626135508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,64,0,1,float16,fp8,0,0.1583573321501414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,64,128,1,fp8,fp8,0,0.21847999095916748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,64,0,1,fp8,fp8,0,0.14336533347765604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,64,128,1,fp8,fp8,0,0.21788267294565836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,64,0,1,fp8,fp8,0,0.14385066429773966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,64,128,1,float16,float16,0,0.16389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,64,0,1,float16,float16,0,0.15956266721089682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,64,128,1,float16,fp8,0,0.16523200273513794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,64,0,1,float16,fp8,0,0.15955199797948202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,64,128,1,float16,float16,0,0.1665386656920115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,64,0,1,float16,float16,0,0.16102932890256247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,64,128,1,float16,fp8,0,0.16732800006866455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,64,128,1,fp8,fp8,0,0.2185279925664266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,64,0,1,fp8,fp8,0,0.14524267117182413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,64,0,1,float16,fp8,0,0.1622719963391622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,64,128,1,float16,float16,0,0.1690346598625183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,64,0,1,float16,float16,0,0.1632480025291443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,64,128,1,float16,fp8,0,0.1695786714553833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,64,128,1,fp8,fp8,0,0.2204266587893168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,64,0,1,float16,fp8,0,0.1641439994176229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,64,0,1,fp8,fp8,0,0.14502933621406555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,64,128,1,float16,float16,0,0.11055999994277954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,64,0,1,float16,float16,0,0.11030399799346924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,64,128,1,float16,fp8,0,0.10795733332633972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,64,128,1,fp8,fp8,0,0.13757866621017456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,64,0,1,float16,fp8,0,0.10706133643786113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,64,0,1,fp8,fp8,0,0.09268266956011455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,64,128,1,float16,float16,0,0.09355733791987102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,64,0,1,float16,float16,0,0.08731200297673543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,64,128,1,float16,fp8,0,0.0941439966360728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,64,128,1,float16,fp8,0,0.09500799576441447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,64,128,1,fp8,fp8,0,0.1251306633154551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,64,0,1,float16,fp8,0,0.08824533224105835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,64,0,1,fp8,fp8,0,0.0809386670589447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,64,128,1,float16,float16,0,0.09613866607348125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,64,128,1,float16,float16,0,0.09298666318257649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,64,0,1,float16,float16,0,0.08832533160845439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,64,128,1,fp8,fp8,0,0.1239946683247884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,64,0,1,float16,fp8,0,0.08924266695976257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,64,0,1,fp8,fp8,0,0.08226666847864787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,64,128,1,float16,float16,0,0.09681600332260132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,64,0,1,float16,float16,0,0.08972799777984619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,64,0,1,float16,float16,0,0.09153599540392558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,64,128,1,float16,fp8,0,0.09493866562843323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,64,128,1,fp8,fp8,0,0.12798933188120523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,64,0,1,float16,fp8,0,0.08941866954167683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,64,0,1,fp8,fp8,0,0.08204266428947449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,64,128,1,float16,fp8,0,0.09755200147628784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,64,128,1,fp8,fp8,0,0.1279146671295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,64,128,1,float16,fp8,0,0.06357866525650024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,64,0,1,float16,fp8,0,0.09152533610661824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,64,128,1,fp8,fp8,0,0.08758399883906047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,64,0,1,fp8,fp8,0,0.08405333757400513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,64,128,1,float16,float16,0,0.06489600241184235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,64,0,1,float16,float16,0,0.059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,64,0,1,float16,fp8,0,0.05881600081920624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,64,128,1,fp8,fp8,0,0.07283199826876323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,64,0,1,fp8,fp8,0,0.05565866827964783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,64,128,1,float16,float16,0,0.055258666475613914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,64,0,1,float16,float16,0,0.051856001218159996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,64,128,1,float16,fp8,0,0.05561600128809611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,64,0,1,float16,fp8,0,0.051829333106676735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,64,0,1,fp8,fp8,0,0.04743466774622599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,64,128,1,float16,float16,0,0.056890666484832764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,64,0,1,float16,float16,0,0.05283733208974203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,64,128,1,float16,float16,0,0.05645333230495453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,64,128,1,float16,fp8,0,0.056277334690093994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,64,128,1,fp8,fp8,0,0.07273600002129872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,64,0,1,float16,fp8,0,0.05241066714127859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,64,0,1,fp8,fp8,0,0.047370667258898415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,64,0,1,fp8,fp8,0,0.04773333172003428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,64,0,1,float16,float16,0,0.05305066704750061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,64,128,1,float16,float16,0,0.058186665177345276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,64,128,1,float16,fp8,0,0.05715733269850413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,64,128,1,fp8,fp8,0,0.0716480016708374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,64,0,1,float16,fp8,0,0.05306133131186167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,64,0,1,float16,float16,0,0.05397866666316986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,64,128,1,float16,fp8,0,0.05782400071620941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,64,128,1,fp8,fp8,0,0.07373866438865662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,64,128,1,float16,fp8,0,0.041221333046754204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,64,0,1,float16,fp8,0,0.05393599967161814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,64,0,1,fp8,fp8,0,0.049269333481788635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,64,128,1,float16,float16,0,0.040864000717798867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,64,0,1,float16,float16,0,0.035317334036032356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,64,128,1,fp8,fp8,0,0.050944000482559204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,64,0,1,float16,fp8,0,0.03522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,64,0,1,fp8,fp8,0,0.032229334115982056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,64,128,1,float16,float16,0,0.03930133332808813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,64,0,1,float16,float16,0,0.033285332222779594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,64,128,1,float16,fp8,0,0.038906666139761605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,64,128,1,fp8,fp8,0,0.04977599779764811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,64,0,1,float16,fp8,0,0.032831999162832894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,64,0,1,fp8,fp8,0,0.030453334252039593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,64,128,1,float16,float16,0,0.03876800090074539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,64,0,1,float16,float16,0,0.03252800057331721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,64,128,1,float16,fp8,0,0.03980266551176707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,64,128,1,fp8,fp8,0,0.0496373325586319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,64,0,1,float16,fp8,0,0.03323200096686681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,64,128,1,fp8,fp8,0,0.05039466420809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,64,0,1,fp8,fp8,0,0.03073599934577942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,64,128,1,float16,float16,0,0.03993066648642222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,64,0,1,float16,float16,0,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,64,128,1,float16,fp8,0,0.039135999977588654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,64,0,1,float16,fp8,0,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,64,0,1,fp8,fp8,0,0.03123733401298523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,64,128,1,float16,float16,0,0.03949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,64,0,1,float16,float16,0,0.0340693344672521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,64,128,1,float16,fp8,0,0.040693332751592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,64,128,1,float16,fp8,0,0.02852799991766612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,64,128,1,fp8,fp8,0,0.05026666820049286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,64,0,1,float16,fp8,0,0.034261333445707955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,64,0,1,fp8,fp8,0,0.023919999599456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,64,0,1,fp8,fp8,0,0.03102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,64,0,1,float16,float16,0,0.023792001108328503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,64,128,1,float16,float16,0,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,64,0,1,float16,float16,0,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,64,128,1,fp8,fp8,0,0.03436266630887985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,64,0,1,float16,fp8,0,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,64,128,1,float16,float16,0,0.026159999271233875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,64,0,1,float16,float16,0,0.02405333270629247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,64,128,1,float16,float16,0,0.026533332963784535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,64,128,1,float16,fp8,0,0.026629333694775898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,64,128,1,fp8,fp8,0,0.032602667808532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,64,0,1,float16,fp8,0,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,64,0,1,fp8,fp8,0,0.02260799954334895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,64,128,1,float16,fp8,0,0.026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,64,128,1,float16,fp8,0,0.026565333207448322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,64,128,1,fp8,fp8,0,0.032885332902272545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,64,0,1,float16,fp8,0,0.0242399995525678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,64,0,1,fp8,fp8,0,0.023706667125225067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,64,128,1,float16,float16,0,0.026533332963784535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,64,0,1,float16,float16,0,0.023919999599456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,64,128,1,fp8,fp8,0,0.033013333876927696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,64,0,1,float16,fp8,0,0.024495999018351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,64,0,1,fp8,fp8,0,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,64,128,1,float16,float16,0,0.026698666314284008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,64,0,1,float16,float16,0,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,64,128,1,float16,fp8,0,0.027509334186712902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,64,128,1,fp8,fp8,0,0.03374933451414108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,64,0,1,float16,fp8,0,0.02477866659561793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,64,0,1,fp8,fp8,0,0.02369066576162974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,64,128,1,float16,float16,0,0.9624106884002686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,64,0,1,float16,float16,0,0.9345013300577799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,64,128,1,float16,fp8,0,0.9683413505554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,64,0,1,float16,fp8,0,0.9397439956665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,64,0,1,fp8,fp8,0,0.8595840136210123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,64,128,1,fp8,fp8,0,1.300976037979126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,64,128,1,float16,float16,0,0.9851733048756918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,64,0,1,float16,float16,0,0.956501324971517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,64,128,1,float16,fp8,0,0.9820266564687093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,64,0,1,fp8,fp8,0,0.8752319812774658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,64,0,1,float16,fp8,0,0.9562666416168213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,64,128,1,fp8,fp8,0,1.3082880179087322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,64,128,1,float16,float16,0,0.9791093667348226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,64,0,1,float16,float16,0,0.9644693533579508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,64,128,1,float16,fp8,0,0.9826133251190186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,64,128,1,fp8,fp8,0,1.319386641184489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,64,0,1,fp8,fp8,0,0.879466692606608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,64,0,1,float16,fp8,0,0.9610719680786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,64,128,1,float16,float16,0,0.9983786741892496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,64,0,1,float16,float16,0,0.9714346726735433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,64,128,1,float16,fp8,0,1.0083626906077068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,64,128,1,fp8,fp8,0,1.3399786949157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,64,0,1,float16,fp8,0,0.9813600381215414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,64,128,1,float16,float16,0,0.6371200084686279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,64,0,1,float16,float16,0,0.6255040168762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,64,128,1,float16,fp8,0,0.6122346719106039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,64,0,1,fp8,fp8,0,0.9123093287150065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,64,128,1,fp8,fp8,0,0.7501386801401774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,64,0,1,float16,fp8,0,0.6009866793950399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,64,0,1,fp8,fp8,0,0.5256640116373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,64,128,1,float16,float16,0,0.49052266279856366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,64,0,1,float16,float16,0,0.47445865472157794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,64,128,1,float16,fp8,0,0.48839465777079266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,64,0,1,float16,fp8,0,0.4787199894587199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,64,0,1,fp8,fp8,0,0.437450647354126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,64,128,1,fp8,fp8,0,0.6678773562113444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,64,128,1,float16,float16,0,0.49668268362681073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,64,0,1,float16,float16,0,0.48421867688496906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,64,128,1,float16,fp8,0,0.4965333143870036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,64,0,1,float16,fp8,0,0.48339200019836426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,64,128,1,fp8,fp8,0,0.6611839930216471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,64,0,1,fp8,fp8,0,0.44952531655629474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,64,128,1,float16,float16,0,0.5073546568552653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,64,0,1,float16,float16,0,0.48456533749898273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,64,128,1,float16,fp8,0,0.49859734376271564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,64,128,1,fp8,fp8,0,0.6648853222529093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,64,0,1,float16,fp8,0,0.4872533480326335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,64,0,1,fp8,fp8,0,0.44566933314005536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,64,128,1,float16,float16,0,0.5158240000406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,64,0,1,float16,float16,0,0.5037599802017212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,64,128,1,float16,fp8,0,0.5136746565500895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,64,0,1,float16,fp8,0,0.4954933325449626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,64,128,1,float16,float16,0,0.33234665791193646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,64,128,1,fp8,fp8,0,0.6706559658050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,64,0,1,float16,float16,0,0.3259253303209941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,64,128,1,fp8,fp8,0,0.3824160099029541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,64,0,1,fp8,fp8,0,0.45196799437205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,64,128,1,float16,fp8,0,0.3253013292948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,64,0,1,float16,fp8,0,0.31894399722417194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,64,0,1,fp8,fp8,0,0.26689600944519043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,64,128,1,float16,float16,0,0.25641600290934247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,64,0,1,float16,float16,0,0.24901332457860312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,64,128,1,float16,fp8,0,0.25594133138656616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,64,128,1,fp8,fp8,0,0.3418560028076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,64,0,1,float16,fp8,0,0.24819733699162802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,64,0,1,fp8,fp8,0,0.234117329120636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,64,128,1,float16,float16,0,0.26232532660166424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,64,0,1,float16,float16,0,0.25271467367808026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,64,128,1,float16,fp8,0,0.25882667303085327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,64,128,1,fp8,fp8,0,0.34033600489298504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,64,0,1,float16,fp8,0,0.252949337164561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,64,0,1,fp8,fp8,0,0.23332266012827554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,64,128,1,float16,float16,0,0.26132800181706745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,64,0,1,float16,float16,0,0.2579786578814189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,64,128,1,float16,fp8,0,0.2632906635602315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,64,128,1,fp8,fp8,0,0.34270934263865155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,64,0,1,float16,fp8,0,0.25566399097442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,64,0,1,fp8,fp8,0,0.23350399732589722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,64,128,1,float16,float16,0,0.2679520050684611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,64,128,1,fp8,fp8,0,0.34836268424987793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,64,0,1,float16,float16,0,0.26362133026123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,64,128,1,float16,fp8,0,0.26851733525594074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,64,0,1,float16,fp8,0,0.2635626594225566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,64,128,1,float16,float16,0,0.1790133317311605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,64,0,1,fp8,fp8,0,0.2355733315149943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,64,0,1,float16,float16,0,0.17778132359186807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,64,128,1,float16,fp8,0,0.17493865887324014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,64,128,1,fp8,fp8,0,0.19900266329447427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,64,0,1,float16,fp8,0,0.173962672551473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,64,0,1,fp8,fp8,0,0.14482133587201437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,64,128,1,float16,float16,0,0.13621866703033447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,64,0,1,float16,float16,0,0.13244799772898355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,64,0,1,float16,float16,0,0.1337440013885498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,64,128,1,float16,fp8,0,0.13497599959373474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,64,128,1,fp8,fp8,0,0.17937066157658896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,64,0,1,float16,fp8,0,0.13316266735394797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,64,0,1,fp8,fp8,0,0.12762133280436197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,64,0,1,fp8,fp8,0,0.12826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,64,128,1,float16,float16,0,0.1372266709804535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,64,128,1,float16,fp8,0,0.13818132877349854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,64,128,1,fp8,fp8,0,0.18203200896581015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,64,0,1,float16,fp8,0,0.13525333007176718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,64,128,1,float16,float16,0,0.13938132921854654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,64,0,1,float16,float16,0,0.1367946664492289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,64,0,1,fp8,fp8,0,0.12939733266830444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,64,128,1,float16,fp8,0,0.1402079959710439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,64,128,1,fp8,fp8,0,0.18281066417694092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,64,0,1,float16,fp8,0,0.13698133826255798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,64,128,1,float16,float16,0,0.14379733800888062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,64,0,1,float16,float16,0,0.14200533429781595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,64,128,1,float16,fp8,0,0.14481066664059958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,64,128,1,fp8,fp8,0,0.18481600284576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,64,0,1,float16,fp8,0,0.14220266540845236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,64,0,1,fp8,fp8,0,0.13083199659983316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,64,0,1,float16,fp8,0,0.09611733754475911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,64,128,1,float16,float16,0,0.09973866740862529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,64,0,1,float16,float16,0,0.09859733780225118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,64,128,1,float16,fp8,0,0.09685867031415303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,64,128,1,fp8,fp8,0,0.11053867141405742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,64,0,1,fp8,fp8,0,0.0846453309059143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,64,128,1,float16,float16,0,0.07323200007279713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,64,0,1,float16,float16,0,0.07203733424345653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,64,128,1,float16,fp8,0,0.07392533123493195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,64,128,1,fp8,fp8,0,0.0990666647752126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,64,0,1,float16,fp8,0,0.07248533268769582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,64,128,1,fp8,fp8,0,0.10046933094660442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,64,0,1,fp8,fp8,0,0.07335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,64,128,1,float16,float16,0,0.07478400071461995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,64,0,1,float16,float16,0,0.07266666491826375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,64,128,1,float16,fp8,0,0.0747519979874293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,64,0,1,float16,fp8,0,0.07270933190981548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,64,0,1,fp8,fp8,0,0.07187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,64,128,1,float16,float16,0,0.07553599774837494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,64,0,1,float16,float16,0,0.07482666770617168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,64,128,1,float16,fp8,0,0.07550933460394542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,64,128,1,fp8,fp8,0,0.10058666268984477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,64,0,1,float16,fp8,0,0.07402133444945018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,64,0,1,fp8,fp8,0,0.07259733478228252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,64,128,1,float16,float16,0,0.07689066727956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,64,0,1,float16,float16,0,0.07690666615962982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,64,128,1,float16,fp8,0,0.07738133271535237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,64,128,1,fp8,fp8,0,0.101583997408549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,64,0,1,float16,fp8,0,0.07576533158620198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,64,0,1,fp8,fp8,0,0.07533333202203114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,64,128,1,float16,float16,0,0.04943466683228811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,64,0,1,float16,float16,0,0.048895999789237976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,64,128,1,float16,fp8,0,0.048207998275756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,64,128,1,fp8,fp8,0,0.06520000100135803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,64,0,1,float16,fp8,0,0.04780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,64,0,1,fp8,fp8,0,0.050111999114354454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,64,128,1,float16,float16,0,0.041850666205088295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,64,0,1,float16,float16,0,0.040949332217375435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,64,128,1,float16,fp8,0,0.04201066493988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,64,128,1,fp8,fp8,0,0.05635199944178263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,64,0,1,float16,fp8,0,0.04179200033346812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,64,0,1,fp8,fp8,0,0.04281599819660187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,64,128,1,float16,float16,0,0.042080000042915344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,64,0,1,float16,float16,0,0.04134399940570196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,64,128,1,float16,fp8,0,0.04228800038496653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,64,128,1,fp8,fp8,0,0.05603733162085215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,64,0,1,float16,fp8,0,0.04165866722663244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,64,0,1,float16,fp8,0,0.041877334316571556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,64,0,1,fp8,fp8,0,0.04278400043646494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,64,128,1,float16,float16,0,0.0432640016078949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,64,0,1,float16,float16,0,0.042303999265034996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,64,128,1,float16,fp8,0,0.042949333786964417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,64,128,1,fp8,fp8,0,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,64,0,1,fp8,fp8,0,0.04294399917125702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,64,128,1,float16,float16,0,0.0435146689414978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,64,0,1,float16,float16,0,0.04247466723124186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,64,128,1,float16,fp8,0,0.043951998154322304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,64,128,1,fp8,fp8,0,0.05708266794681549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,64,0,1,float16,fp8,0,0.04269866645336151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,64,0,1,fp8,fp8,0,0.04324266811211904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,64,128,1,float16,float16,0,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,64,0,1,float16,float16,0,0.03311466674009959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,64,128,1,float16,fp8,0,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,64,128,1,fp8,fp8,0,0.035904000202814736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,64,0,1,float16,fp8,0,0.03278933217128118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,64,0,1,fp8,fp8,0,0.029648000995318096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,64,0,1,fp8,fp8,0,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,64,128,1,float16,float16,0,0.03002133220434189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,64,0,1,float16,float16,0,0.029722665747006733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,64,128,1,float16,fp8,0,0.0305226668715477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,64,128,1,fp8,fp8,0,0.034789333740870156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,64,0,1,float16,fp8,0,0.03028800090154012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,64,128,1,float16,float16,0,0.030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,64,0,1,float16,float16,0,0.030495998760064442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,64,0,1,float16,float16,0,0.03090133269627889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,64,128,1,float16,fp8,0,0.030005333324273426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,64,128,1,fp8,fp8,0,0.0345920001467069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,64,0,1,float16,fp8,0,0.029887999097506206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,64,0,1,fp8,fp8,0,0.028170667588710785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,64,128,1,float16,float16,0,0.03107200066248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,64,128,1,float16,fp8,0,0.03149333347876867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,64,128,1,fp8,fp8,0,0.035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,64,0,1,float16,fp8,0,0.030970667799313862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,64,0,1,fp8,fp8,0,0.028037334481875103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,64,128,1,float16,float16,0,0.03123733401298523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,64,0,1,float16,float16,0,0.030586667358875275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,64,128,1,float16,fp8,0,0.031712000568707786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,64,128,1,fp8,fp8,0,0.03510933369398117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,64,0,1,float16,fp8,0,0.031023999055226643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,64,0,1,fp8,fp8,0,0.029120000700155895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,64,128,1,float16,float16,0,0.02203733225663503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,64,0,1,float16,float16,0,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,64,128,1,float16,fp8,0,0.02179733415444692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,64,128,1,fp8,fp8,0,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,64,0,1,float16,fp8,0,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,64,128,1,fp8,fp8,0,0.024106666445732117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,64,0,1,fp8,fp8,0,0.02187199890613556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,64,128,1,float16,float16,0,0.020261333634455998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,64,0,1,float16,float16,0,0.01985599969824155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,64,128,1,float16,fp8,0,0.020629333953062694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,64,0,1,float16,fp8,0,0.019989332805077236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,64,0,1,fp8,fp8,0,0.020128000527620316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,64,128,1,float16,float16,0,0.020400000115235645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,64,0,1,fp8,fp8,0,0.02038399999340375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,64,0,1,float16,float16,0,0.02015999952952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,64,0,1,float16,float16,0,0.020053333292404812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,64,128,1,float16,fp8,0,0.020501332978407543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,64,128,1,fp8,fp8,0,0.02385599911212921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,64,0,1,float16,fp8,0,0.020213333268960316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,64,128,1,float16,float16,0,0.02059200033545494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,64,128,1,float16,fp8,0,0.020794666061798733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,64,128,1,fp8,fp8,0,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,64,0,1,float16,fp8,0,0.02029866725206375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,64,128,1,fp8,fp8,0,0.02492800106604894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,64,0,1,float16,fp8,0,0.020576000213623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,64,0,1,fp8,fp8,0,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,64,0,1,fp8,fp8,0,0.020848001043001812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,64,128,1,float16,float16,0,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,64,0,1,float16,float16,0,0.02067199970285098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,64,128,1,float16,fp8,0,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,64,128,1,float16,float16,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,64,0,1,float16,float16,0,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,64,128,1,float16,fp8,0,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,64,128,1,fp8,fp8,0,0.022634667654832203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,64,0,1,float16,fp8,0,0.01815466706951459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,64,0,1,fp8,fp8,0,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,64,128,1,float16,float16,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,64,0,1,float16,float16,0,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,64,128,1,float16,float16,0,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,64,128,1,float16,fp8,0,0.016741332908471424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,64,128,1,fp8,fp8,0,0.023007998863856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,64,0,1,float16,fp8,0,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,64,0,1,fp8,fp8,0,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,64,0,1,float16,float16,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,64,128,1,float16,fp8,0,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,64,0,1,float16,float16,0,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,64,128,1,fp8,fp8,0,0.02253866692384084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,64,128,1,fp8,fp8,0,0.022965334355831146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,64,0,1,float16,fp8,0,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,64,0,1,fp8,fp8,0,0.019632000476121902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,64,128,1,float16,float16,0,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,64,128,1,float16,fp8,0,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,64,0,1,float16,fp8,0,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,64,0,1,fp8,fp8,0,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,64,128,1,float16,float16,0,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,64,0,1,float16,float16,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,64,128,1,float16,fp8,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,64,128,1,fp8,fp8,0,0.022863999009132385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,64,0,1,float16,fp8,0,0.016778666526079178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,64,0,1,fp8,fp8,0,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,64,128,1,float16,float16,0,0.35807998975118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,64,0,1,float16,float16,0,0.35788798332214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,64,128,1,float16,fp8,0,0.35598933696746826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,64,128,1,fp8,fp8,0,0.43273067474365234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,64,0,1,float16,fp8,0,0.3574346701304118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,64,0,1,fp8,fp8,0,0.4339359998703003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,64,128,1,float16,float16,0,0.3659093379974365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,64,0,1,float16,float16,0,0.3667360146840413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,64,128,1,float16,fp8,0,0.3663146495819092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,64,128,1,fp8,fp8,0,0.4403093258539836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,64,0,1,float16,fp8,0,0.3649493455886841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,64,0,1,fp8,fp8,0,0.44092265764872235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,64,128,1,float16,float16,0,0.36739198366800946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,64,0,1,float16,float16,0,0.3674986759821574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,64,128,1,float16,fp8,0,0.3668479919433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,64,0,1,float16,fp8,0,0.36726399262746173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,64,128,1,fp8,fp8,0,0.44223999977111816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,64,0,1,fp8,fp8,0,0.4416373173395793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,64,128,1,float16,float16,0,0.37971198558807373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,64,0,1,float16,float16,0,0.3797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,64,128,1,float16,fp8,0,0.3757760127385457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,64,128,1,fp8,fp8,0,0.46538134415944415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,64,0,1,float16,fp8,0,0.37598931789398193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,64,128,1,float16,float16,0,0.27449599901835126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,64,0,1,fp8,fp8,0,0.4666506846745809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,64,0,1,float16,float16,0,0.2762133280436198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,64,128,1,float16,fp8,0,0.26450133323669434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,64,128,1,fp8,fp8,0,0.26927467187245685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,64,0,1,float16,float16,0,0.187226672967275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,64,0,1,float16,fp8,0,0.2661919991175334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,64,0,1,fp8,fp8,0,0.2661386728286743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,64,128,1,float16,float16,0,0.18789867560068765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,64,128,1,float16,fp8,0,0.18790932496388754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,64,128,1,fp8,fp8,0,0.22952532768249512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,64,0,1,float16,fp8,0,0.18756266434987387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,64,0,1,fp8,fp8,0,0.22886399428049722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,64,128,1,float16,float16,0,0.19107200702031454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,64,0,1,float16,float16,0,0.19062399864196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,64,128,1,float16,fp8,0,0.1912426749865214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,64,128,1,fp8,fp8,0,0.2312906583150228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,64,0,1,float16,fp8,0,0.1913493275642395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,64,0,1,fp8,fp8,0,0.23178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,64,128,1,float16,float16,0,0.1930826703707377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,64,0,1,float16,float16,0,0.19340799252192178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,64,128,1,float16,fp8,0,0.19272534052530924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,64,128,1,fp8,fp8,0,0.2312693397204081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,64,0,1,float16,fp8,0,0.19242133696873984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,64,0,1,fp8,fp8,0,0.23076266050338745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,64,128,1,float16,float16,0,0.20110400517781576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,64,0,1,float16,float16,0,0.19912532965342203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,64,128,1,float16,fp8,0,0.19857599337895712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,64,0,1,float16,fp8,0,0.19882667064666748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,64,128,1,fp8,fp8,0,0.23772267500559488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,64,128,1,float16,float16,0,0.14797866344451904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,64,0,1,fp8,fp8,0,0.23530133565266928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,64,0,1,float16,float16,0,0.1478613317012787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,64,128,1,float16,fp8,0,0.14260266224543253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,64,128,1,fp8,fp8,0,0.14386666814486185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,64,0,1,float16,fp8,0,0.14309333761533102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,64,0,1,fp8,fp8,0,0.14359999696413675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,64,128,1,float16,float16,0,0.102101335922877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,64,0,1,float16,fp8,0,0.1016373336315155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,64,0,1,float16,float16,0,0.10116266210873921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,64,128,1,float16,fp8,0,0.10204799969991048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,64,128,1,fp8,fp8,0,0.12586667140324911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,64,0,1,fp8,fp8,0,0.12652800480524698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,64,128,1,float16,float16,0,0.10290666421254475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,64,0,1,float16,float16,0,0.10268266995747884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,64,128,1,float16,fp8,0,0.10271466771761577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,64,128,1,fp8,fp8,0,0.12668800354003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,64,0,1,float16,fp8,0,0.10299733281135559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,64,0,1,fp8,fp8,0,0.1269493301709493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,64,128,1,float16,float16,0,0.10501866539319356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,64,0,1,float16,float16,0,0.10442666212717693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,64,0,1,fp8,fp8,0,0.1288640002409617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,64,128,1,float16,fp8,0,0.10474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,64,128,1,fp8,fp8,0,0.1285760005315145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,64,0,1,float16,float16,0,0.10924800237019856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,64,0,1,float16,fp8,0,0.10496532917022705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,64,128,1,float16,float16,0,0.10902399818102519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,64,128,1,float16,fp8,0,0.10755200187365214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,64,128,1,fp8,fp8,0,0.12982933719952902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,64,0,1,float16,fp8,0,0.10809600353240967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,64,0,1,fp8,fp8,0,0.1297546625137329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,64,128,1,float16,float16,0,0.07991466422875722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,64,0,1,float16,float16,0,0.080485333998998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,64,128,1,float16,fp8,0,0.07772799829641978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,64,128,1,fp8,fp8,0,0.08356266220410664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,64,0,1,float16,fp8,0,0.07834133505821228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,64,0,1,fp8,fp8,0,0.08326399823029836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,64,128,1,float16,float16,0,0.056218668818473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,64,0,1,float16,float16,0,0.056143999099731445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,64,128,1,float16,fp8,0,0.0566293348868688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,64,128,1,fp8,fp8,0,0.07190399865309398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,64,0,1,float16,fp8,0,0.056517332792282104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,64,0,1,fp8,fp8,0,0.0728000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,64,0,1,float16,fp8,0,0.05666666726271311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,64,128,1,float16,float16,0,0.056517332792282104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,64,0,1,float16,float16,0,0.05648000041643778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,64,128,1,float16,fp8,0,0.05716800192991892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,64,128,1,float16,fp8,0,0.056847999493281044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,64,128,1,fp8,fp8,0,0.07182933390140533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,64,0,1,fp8,fp8,0,0.07267199953397115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,64,128,1,float16,float16,0,0.05756799876689911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,64,0,1,float16,float16,0,0.058117335041364036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,64,128,1,fp8,fp8,0,0.07310399909814198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,64,0,1,float16,fp8,0,0.05709333221117655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,64,0,1,float16,fp8,0,0.05957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,64,0,1,fp8,fp8,0,0.07268799841403961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,64,128,1,float16,float16,0,0.04028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,64,128,1,float16,float16,0,0.05942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,64,0,1,float16,float16,0,0.059546664357185364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,64,128,1,float16,fp8,0,0.0602453351020813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,64,128,1,fp8,fp8,0,0.07467199862003326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,64,0,1,fp8,fp8,0,0.0746559997399648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,64,128,1,float16,float16,0,0.03316800047953924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,64,0,1,float16,float16,0,0.03264000018437704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,64,0,1,float16,float16,0,0.04035199930270513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,64,128,1,float16,fp8,0,0.03896533449490865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,64,128,1,fp8,fp8,0,0.05019199848175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,64,0,1,float16,fp8,0,0.039647998909155525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,64,0,1,fp8,fp8,0,0.04927466809749603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,64,0,1,float16,float16,0,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,64,128,1,float16,fp8,0,0.03294399877389272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,64,128,1,fp8,fp8,0,0.04331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,64,0,1,float16,fp8,0,0.032816000282764435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,64,0,1,fp8,fp8,0,0.043050666650136314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,64,128,1,float16,float16,0,0.03292266776164373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,64,128,1,float16,fp8,0,0.03324266771475474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,64,128,1,float16,fp8,0,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,64,128,1,fp8,fp8,0,0.04214933514595032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,64,0,1,float16,fp8,0,0.03341866781314214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,64,0,1,fp8,fp8,0,0.04274666806062063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,64,128,1,float16,float16,0,0.033733333150545754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,64,0,1,float16,float16,0,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,64,128,1,fp8,fp8,0,0.043568000197410583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,64,0,1,float16,fp8,0,0.03385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,64,0,1,fp8,fp8,0,0.0432586669921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,64,128,1,float16,float16,0,0.034048000971476235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,64,0,1,float16,float16,0,0.03437866767247518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,64,128,1,float16,fp8,0,0.034416000048319496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,64,128,1,fp8,fp8,0,0.043653334180514015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,64,0,1,float16,fp8,0,0.034586665530999504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,64,0,1,fp8,fp8,0,0.04346133271853129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,64,128,1,float16,float16,0,0.02481599897146225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,64,0,1,float16,float16,0,0.02465066562096278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,64,128,1,float16,fp8,0,0.02456533412138621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,64,128,1,fp8,fp8,0,0.028677334388097126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,64,0,1,float16,fp8,0,0.02446399877468745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,64,0,1,fp8,fp8,0,0.029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,64,128,1,float16,float16,0,0.022453332940737408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,64,0,1,float16,float16,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,64,128,1,float16,fp8,0,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,64,128,1,fp8,fp8,0,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,64,0,1,float16,fp8,0,0.022458667556444805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,64,0,1,fp8,fp8,0,0.028069332242012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,64,128,1,float16,float16,0,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,64,0,1,float16,float16,0,0.022437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,64,128,1,float16,fp8,0,0.02272533377011617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,64,128,1,fp8,fp8,0,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,64,128,1,float16,fp8,0,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,64,128,1,fp8,fp8,0,0.028165332973003387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,64,0,1,float16,fp8,0,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,64,0,1,fp8,fp8,0,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,64,128,1,float16,float16,0,0.022986667851607006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,64,0,1,float16,float16,0,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,64,0,1,float16,fp8,0,0.02362666775782903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,64,0,1,fp8,fp8,0,0.027834666272004444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,64,128,1,float16,float16,0,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,64,0,1,float16,float16,0,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,64,128,1,float16,fp8,0,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,64,128,1,fp8,fp8,0,0.029258665939172108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,64,0,1,float16,fp8,0,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,64,0,1,fp8,fp8,0,0.02845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,64,128,1,float16,float16,0,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,64,0,1,float16,float16,0,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,64,128,1,float16,fp8,0,0.018810667097568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,64,128,1,fp8,fp8,0,0.021530665457248688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,64,0,1,float16,fp8,0,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,64,0,1,fp8,fp8,0,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,64,128,1,float16,float16,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,64,0,1,float16,float16,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,64,128,1,float16,float16,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,64,128,1,float16,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,64,128,1,fp8,fp8,0,0.020629333953062694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,64,0,1,float16,fp8,0,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,64,0,1,fp8,fp8,0,0.020554666717847187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,64,0,1,float16,float16,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,64,128,1,float16,fp8,0,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,64,128,1,fp8,fp8,0,0.02040533348917961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,64,0,1,float16,fp8,0,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,64,0,1,fp8,fp8,0,0.020661332954963047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,64,128,1,float16,float16,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,64,0,1,float16,fp8,0,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,64,0,1,float16,float16,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,64,128,1,float16,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,64,128,1,fp8,fp8,0,0.02070933332045873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,64,0,1,fp8,fp8,0,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,64,128,1,float16,float16,0,0.018250666558742523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,64,0,1,float16,float16,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,64,128,1,float16,fp8,0,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,64,0,1,float16,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,64,128,1,fp8,fp8,0,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,64,0,1,fp8,fp8,0,0.020869334538777668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,64,128,1,float16,float16,0,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,64,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,64,128,1,float16,fp8,0,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,64,128,1,fp8,fp8,0,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,64,0,1,float16,fp8,0,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,64,0,1,fp8,fp8,0,0.019541333119074505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,64,128,1,float16,float16,0,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,64,0,1,float16,float16,0,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,64,128,1,float16,fp8,0,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,64,128,1,fp8,fp8,0,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,64,0,1,float16,fp8,0,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,64,0,1,fp8,fp8,0,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,64,128,1,float16,float16,0,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,64,0,1,float16,float16,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,64,128,1,float16,fp8,0,0.014618666221698126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,64,128,1,fp8,fp8,0,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,64,128,1,fp8,fp8,0,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,64,0,1,float16,fp8,0,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,64,0,1,fp8,fp8,0,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,64,128,1,float16,float16,0,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,64,128,1,float16,float16,0,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,64,0,1,float16,float16,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,64,128,1,float16,fp8,0,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,64,0,1,float16,fp8,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,64,0,1,fp8,fp8,0,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,64,128,1,float16,fp8,0,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,64,128,1,fp8,fp8,0,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,64,0,1,float16,fp8,0,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,64,0,1,float16,float16,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,64,0,1,fp8,fp8,0,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,64,128,1,float16,float16,0,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,64,0,1,float16,float16,0,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,64,128,1,float16,fp8,0,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,64,128,1,fp8,fp8,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,64,0,1,float16,fp8,0,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,64,0,1,fp8,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,64,128,1,float16,float16,0,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,64,0,1,float16,float16,0,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,64,128,1,float16,fp8,0,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,64,128,1,fp8,fp8,0,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,64,0,1,float16,fp8,0,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,64,0,1,fp8,fp8,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,64,128,1,float16,float16,0,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,64,0,1,float16,float16,0,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,64,128,1,float16,fp8,0,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,64,128,1,fp8,fp8,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,64,0,1,float16,fp8,0,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,64,0,1,fp8,fp8,0,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,64,128,1,float16,float16,0,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,64,0,1,float16,float16,0,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,64,128,1,float16,fp8,0,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,64,128,1,fp8,fp8,0,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,64,0,1,float16,fp8,0,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,64,0,1,fp8,fp8,0,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,64,128,1,float16,float16,0,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,64,0,1,float16,float16,0,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,64,128,1,float16,fp8,0,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,64,128,1,fp8,fp8,0,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,64,0,1,float16,fp8,0,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,64,0,1,fp8,fp8,0,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,64,128,1,float16,float16,0,0.17383466164271036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,64,0,1,float16,float16,0,0.17381866772969565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,64,128,1,float16,fp8,0,0.1737226645151774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,64,0,1,float16,fp8,0,0.17340266704559326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,64,128,1,fp8,fp8,0,0.30646934111913043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,64,0,1,fp8,fp8,0,0.3070506652196248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,64,128,1,float16,float16,0,0.17769600947697958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,64,0,1,float16,float16,0,0.17747733990351358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,64,128,1,float16,fp8,0,0.17757334311803183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,64,0,1,float16,fp8,0,0.17691733439763388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,64,128,1,fp8,fp8,0,0.3086026708285014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,64,0,1,fp8,fp8,0,0.30855466922124225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,64,128,1,float16,float16,0,0.1795253356297811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,64,0,1,float16,float16,0,0.17870400349299112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,64,128,1,float16,fp8,0,0.17679999272028604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,64,128,1,fp8,fp8,0,0.3105706572532654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,64,0,1,fp8,fp8,0,0.3089173237482707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,64,128,1,float16,float16,0,0.18684800465901694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,64,0,1,float16,fp8,0,0.1774346629778544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,64,0,1,float16,float16,0,0.18606932957967123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,64,128,1,float16,fp8,0,0.18572266896565756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,64,0,1,float16,fp8,0,0.1856480042139689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,64,128,1,fp8,fp8,0,0.31876800457636517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,64,128,1,float16,float16,0,0.14411733547846475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,64,0,1,float16,float16,0,0.14437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,64,0,1,fp8,fp8,0,0.319157342116038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,64,128,1,float16,fp8,0,0.13904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,64,128,1,fp8,fp8,0,0.18438933293024698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,64,0,1,float16,fp8,0,0.1385599970817566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,64,0,1,fp8,fp8,0,0.18290134270985922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,64,128,1,float16,float16,0,0.0944106678167979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,64,0,1,fp8,fp8,0,0.1662879983584086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,64,0,1,float16,float16,0,0.09518933296203613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,64,128,1,float16,fp8,0,0.09494933485984802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,64,128,1,fp8,fp8,0,0.16447466611862183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,64,0,1,float16,fp8,0,0.09523733456929524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,64,128,1,float16,float16,0,0.09642133116722107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,64,0,1,float16,float16,0,0.09655466675758362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,64,128,1,float16,fp8,0,0.09620799620946248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,64,128,1,fp8,fp8,0,0.16693333784739176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,64,0,1,float16,fp8,0,0.09618666768074036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,64,0,1,fp8,fp8,0,0.16659733653068542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,64,128,1,float16,float16,0,0.0978559950987498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,64,0,1,float16,float16,0,0.09815999865531921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,64,128,1,float16,fp8,0,0.09834133585294087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,64,128,1,fp8,fp8,0,0.16664000352223715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,64,0,1,float16,fp8,0,0.09764800469080608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,64,0,1,fp8,fp8,0,0.16752533117930093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,64,128,1,float16,float16,0,0.10181333621342976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,64,0,1,float16,float16,0,0.1018933355808258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,64,128,1,float16,float16,0,0.0764160007238388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,64,128,1,float16,fp8,0,0.10079466303189595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,64,128,1,fp8,fp8,0,0.1704960068066915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,64,128,1,fp8,fp8,0,0.10201066732406616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,64,0,1,float16,fp8,0,0.10131200154622395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,64,0,1,fp8,fp8,0,0.169978658358256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,64,0,1,float16,float16,0,0.0764160007238388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,64,128,1,float16,fp8,0,0.07344533503055573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,64,0,1,float16,fp8,0,0.07441066702206929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,64,0,1,fp8,fp8,0,0.10181867082913716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,64,128,1,float16,float16,0,0.05231999854246775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,64,0,1,float16,float16,0,0.052784000833829246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,64,128,1,float16,fp8,0,0.05276266733805338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,64,128,1,fp8,fp8,0,0.09191999832789104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,64,0,1,float16,fp8,0,0.052832002441088356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,64,0,1,float16,fp8,0,0.05338666836420695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,64,0,1,fp8,fp8,0,0.09302399555842082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,64,128,1,float16,float16,0,0.05351999898751577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,64,128,1,float16,float16,0,0.0536053329706192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,64,0,1,float16,float16,0,0.05317866802215576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,64,128,1,float16,fp8,0,0.052757332722345986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,64,128,1,fp8,fp8,0,0.09245333075523376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,64,0,1,fp8,fp8,0,0.09245866537094116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,64,0,1,float16,float16,0,0.05397866666316986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,64,128,1,float16,fp8,0,0.05402133365472158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,64,128,1,fp8,fp8,0,0.09289066990216573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,64,0,1,float16,fp8,0,0.05346133311589559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,64,0,1,fp8,fp8,0,0.09258133172988892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,64,0,1,float16,fp8,0,0.05563733478387197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,64,128,1,float16,float16,0,0.05606399973233541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,64,0,1,float16,float16,0,0.05622933308283488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,64,128,1,float16,fp8,0,0.05712533493836721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,64,128,1,fp8,fp8,0,0.09453866879145305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,64,128,1,fp8,fp8,0,0.06003733476003011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,64,0,1,fp8,fp8,0,0.09472533067067464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,64,128,1,float16,float16,0,0.03889599939187368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,64,0,1,float16,float16,0,0.03892799963553747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,64,0,1,float16,float16,0,0.03158933420976003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,64,128,1,float16,fp8,0,0.03761066744724909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,64,0,1,float16,fp8,0,0.03812266637881597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,64,0,1,fp8,fp8,0,0.06017066538333893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,64,128,1,float16,float16,0,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,64,128,1,float16,fp8,0,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,64,0,1,float16,float16,0,0.0317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,64,128,1,fp8,fp8,0,0.053472002347310386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,64,0,1,float16,fp8,0,0.03180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,64,0,1,fp8,fp8,0,0.05310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,64,128,1,float16,float16,0,0.032042667269706726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,64,128,1,float16,fp8,0,0.03162666658560435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,64,128,1,fp8,fp8,0,0.05328000088532766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,64,0,1,float16,fp8,0,0.03181866556406021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,64,0,1,fp8,fp8,0,0.053557331363360085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,64,0,1,float16,fp8,0,0.03204799940188726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,64,128,1,float16,float16,0,0.03222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,64,0,1,float16,float16,0,0.03218133250872294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,64,128,1,float16,fp8,0,0.032229334115982056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,64,128,1,fp8,fp8,0,0.053114667534828186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,64,0,1,fp8,fp8,0,0.05345066885153452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,64,128,1,float16,float16,0,0.033215999603271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,64,0,1,fp8,fp8,0,0.053413331508636475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,64,128,1,float16,float16,0,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,64,0,1,float16,float16,0,0.032730666299661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,64,128,1,float16,fp8,0,0.022341333329677582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,64,128,1,float16,fp8,0,0.03257066756486893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,64,128,1,fp8,fp8,0,0.05402133365472158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,64,0,1,float16,fp8,0,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,64,0,1,float16,float16,0,0.022874665757020313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,64,128,1,fp8,fp8,0,0.034314667185147606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,64,0,1,float16,fp8,0,0.022202665607134502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,64,0,1,fp8,fp8,0,0.03446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,64,0,1,float16,fp8,0,0.020768000433842342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,64,0,1,fp8,fp8,0,0.03332799921433131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,64,128,1,float16,float16,0,0.02081599955757459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,64,0,1,float16,float16,0,0.0207893339296182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,64,0,1,float16,float16,0,0.02067199970285098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,64,128,1,float16,fp8,0,0.020917333662509918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,64,128,1,fp8,fp8,0,0.03310399999221166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,64,128,1,fp8,fp8,0,0.03324799984693527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,64,128,1,float16,float16,0,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,64,128,1,float16,fp8,0,0.02089066555102666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,64,0,1,float16,fp8,0,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,64,128,1,fp8,fp8,0,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,64,0,1,fp8,fp8,0,0.033039999504884086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,64,128,1,float16,float16,0,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,64,0,1,float16,float16,0,0.021562665700912476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,64,128,1,float16,fp8,0,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,64,0,1,float16,fp8,0,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,64,0,1,fp8,fp8,0,0.03395200024048487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,64,128,1,float16,float16,0,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,64,0,1,float16,float16,0,0.021514666577180225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,64,128,1,float16,fp8,0,0.021802666286627453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,64,128,1,fp8,fp8,0,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,64,0,1,float16,fp8,0,0.022015998760859173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,64,0,1,fp8,fp8,0,0.0341333324710528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,64,128,1,float16,float16,0,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,64,0,1,float16,float16,0,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,64,0,1,float16,float16,0,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,64,128,1,float16,fp8,0,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,64,128,1,fp8,fp8,0,0.024277334411938984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,64,0,1,float16,fp8,0,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,64,0,1,fp8,fp8,0,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,64,128,1,float16,float16,0,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,64,128,1,float16,fp8,0,0.016629333297411602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,64,128,1,fp8,fp8,0,0.023936000963052113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,64,0,1,float16,fp8,0,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,64,0,1,fp8,fp8,0,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,64,128,1,float16,float16,0,0.01594666639963786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,64,0,1,float16,float16,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,64,128,1,float16,fp8,0,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,64,128,1,fp8,fp8,0,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,64,0,1,float16,fp8,0,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,64,0,1,fp8,fp8,0,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,64,128,1,float16,float16,0,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,64,0,1,float16,float16,0,0.016528000434239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,64,0,1,float16,float16,0,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,64,128,1,float16,fp8,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,64,128,1,fp8,fp8,0,0.02399466683467229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,64,0,1,float16,fp8,0,0.016789333273967106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,64,0,1,fp8,fp8,0,0.023941333095232647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,64,128,1,float16,float16,0,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,64,128,1,float16,fp8,0,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,64,128,1,fp8,fp8,0,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,64,128,1,fp8,fp8,0,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,64,0,1,float16,fp8,0,0.016789333273967106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,64,0,1,fp8,fp8,0,0.024069334069887798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,64,128,1,float16,float16,0,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,64,0,1,float16,float16,0,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,64,128,1,float16,fp8,0,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,64,0,1,float16,fp8,0,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,64,0,1,fp8,fp8,0,0.01970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,64,128,1,float16,float16,0,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,64,0,1,float16,float16,0,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,64,128,1,float16,fp8,0,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,64,128,1,fp8,fp8,0,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,64,0,1,float16,fp8,0,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,64,0,1,fp8,fp8,0,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,64,128,1,float16,float16,0,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,64,0,1,float16,float16,0,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,64,128,1,float16,fp8,0,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,64,128,1,fp8,fp8,0,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,64,0,1,float16,fp8,0,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,64,0,1,fp8,fp8,0,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,64,128,1,float16,float16,0,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,64,0,1,float16,float16,0,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,64,128,1,float16,fp8,0,0.013946666071812311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,64,0,1,float16,float16,0,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,64,128,1,float16,fp8,0,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,64,128,1,fp8,fp8,0,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,64,0,1,float16,fp8,0,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,64,0,1,fp8,fp8,0,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,64,128,1,float16,float16,0,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,64,128,1,fp8,fp8,0,0.019706666469573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,64,0,1,float16,fp8,0,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,64,0,1,fp8,fp8,0,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,64,128,1,float16,float16,0,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,64,0,1,float16,float16,0,0.011930666863918304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,64,128,1,float16,fp8,0,0.01227733368674914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,64,128,1,fp8,fp8,0,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,64,0,1,float16,fp8,0,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,64,0,1,fp8,fp8,0,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,64,128,1,float16,float16,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,64,0,1,float16,float16,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,64,128,1,float16,fp8,0,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,64,128,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,64,0,1,float16,fp8,0,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,64,0,1,fp8,fp8,0,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,64,128,1,float16,float16,0,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,64,0,1,float16,float16,0,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,64,128,1,float16,fp8,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,64,128,1,fp8,fp8,0,0.018165333817402523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,64,0,1,float16,fp8,0,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,64,128,1,fp8,fp8,0,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,64,0,1,float16,fp8,0,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,64,0,1,fp8,fp8,0,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,64,128,1,float16,float16,0,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,64,0,1,float16,float16,0,0.012298667182525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,64,128,1,float16,float16,0,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,64,0,1,float16,float16,0,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,64,0,1,float16,fp8,0,0.012421333541472753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,64,128,1,float16,fp8,0,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,64,128,1,float16,float16,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,64,0,1,fp8,fp8,0,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,64,128,1,float16,fp8,0,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,64,128,1,fp8,fp8,0,0.018432000031073887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,64,0,1,fp8,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,64,0,1,float16,float16,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,64,128,1,float16,fp8,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,64,128,1,fp8,fp8,0,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,64,0,1,float16,fp8,0,0.012234666695197424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,64,0,1,fp8,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,64,128,1,float16,float16,0,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,64,0,1,float16,float16,0,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,64,128,1,float16,fp8,0,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,64,128,1,fp8,fp8,0,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,64,0,1,float16,fp8,0,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,64,0,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,64,128,1,float16,float16,0,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,64,0,1,float16,float16,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,64,128,1,float16,fp8,0,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,64,128,1,fp8,fp8,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,64,128,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,64,0,1,float16,fp8,0,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,64,0,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,64,128,1,float16,float16,0,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,64,0,1,float16,float16,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,64,128,1,float16,fp8,0,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,64,0,1,float16,fp8,0,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,64,0,1,fp8,fp8,0,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,64,128,1,float16,float16,0,0.012026666353146235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,64,0,1,float16,float16,0,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,64,128,1,float16,fp8,0,0.012159999459981918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,64,128,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,64,0,1,float16,fp8,0,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,64,128,1,float16,fp8,0,0.10508267084757487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,64,0,1,fp8,fp8,0,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,64,0,1,float16,fp8,0,0.10537599523862202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,64,128,1,float16,float16,0,0.10517332951227824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,64,0,1,float16,float16,0,0.10493333141009013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,64,128,1,fp8,fp8,0,0.2561759948730469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,64,0,1,fp8,fp8,0,0.2566133340199788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,64,128,1,float16,float16,0,0.10755200187365214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,64,0,1,float16,float16,0,0.1069653332233429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,64,128,1,float16,fp8,0,0.10662933190663655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,64,0,1,float16,fp8,0,0.10664000113805135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,64,128,1,fp8,fp8,0,0.25780800978342694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,64,0,1,fp8,fp8,0,0.25762667258580524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,64,128,1,float16,float16,0,0.10918399691581726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,64,0,1,float16,float16,0,0.10939199725786845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,64,128,1,float16,fp8,0,0.10917333761850993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,64,0,1,float16,fp8,0,0.10982933640480042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,64,0,1,fp8,fp8,0,0.2582506736119588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,64,128,1,fp8,fp8,0,0.25838400920232135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,64,128,1,float16,float16,0,0.11541866262753804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,64,0,1,float16,float16,0,0.11392533779144287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,64,128,1,float16,fp8,0,0.11326932907104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,64,128,1,fp8,fp8,0,0.26266666253407794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,64,0,1,float16,fp8,0,0.1141919990380605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,64,128,1,float16,float16,0,0.0766186664501826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,64,128,1,fp8,fp8,0,0.14964800079663595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,64,0,1,float16,fp8,0,0.07412800192832947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,64,0,1,float16,float16,0,0.07679466903209686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,64,0,1,fp8,fp8,0,0.26176534096399945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,64,128,1,float16,fp8,0,0.0738560010989507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,64,0,1,fp8,fp8,0,0.14935466647148132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,64,128,1,float16,float16,0,0.05761066575845083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,64,0,1,float16,float16,0,0.05754133562246958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,64,128,1,float16,fp8,0,0.05751466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,64,128,1,fp8,fp8,0,0.1386293371518453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,64,0,1,float16,float16,0,0.058277333776156105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,64,0,1,float16,fp8,0,0.05760533114274343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,64,0,1,fp8,fp8,0,0.13829333583513895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,64,128,1,float16,float16,0,0.057578667998313904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,64,128,1,float16,fp8,0,0.05769066512584686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,64,128,1,fp8,fp8,0,0.13896000385284424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,64,0,1,float16,fp8,0,0.05781333148479462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,64,0,1,fp8,fp8,0,0.13852799932161966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,64,128,1,fp8,fp8,0,0.13894933462142944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,64,128,1,float16,float16,0,0.05855466425418854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,64,0,1,float16,float16,0,0.058703998724619545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,64,0,1,fp8,fp8,0,0.1381706694761912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,64,128,1,float16,fp8,0,0.058320000767707825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,64,0,1,float16,fp8,0,0.059088001648585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,64,128,1,float16,float16,0,0.061119998494784035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,64,128,1,fp8,fp8,0,0.14274133245150247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,64,0,1,float16,float16,0,0.06028266747792562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,64,128,1,float16,fp8,0,0.06070933242638906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,64,0,1,float16,float16,0,0.03885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,64,0,1,float16,fp8,0,0.060378665725390114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,64,0,1,fp8,fp8,0,0.14155733585357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,64,128,1,float16,float16,0,0.03867733230193456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,64,128,1,float16,fp8,0,0.03700799991687139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,64,128,1,fp8,fp8,0,0.08321066697438557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,64,0,1,float16,fp8,0,0.03732266773780187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,64,0,1,fp8,fp8,0,0.08498133222262065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,64,0,1,float16,fp8,0,0.03267733256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,64,128,1,float16,float16,0,0.03271999955177307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,64,0,1,float16,float16,0,0.032560000816980995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,64,128,1,float16,fp8,0,0.03331733246644338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,64,128,1,float16,fp8,0,0.03313600023587545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,64,128,1,fp8,fp8,0,0.07656533519426982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,64,0,1,fp8,fp8,0,0.07669333120187123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,64,128,1,float16,float16,0,0.03291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,64,0,1,float16,float16,0,0.032831999162832894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,64,0,1,float16,float16,0,0.03295466552178065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,64,128,1,fp8,fp8,0,0.07614933451016744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,64,0,1,float16,fp8,0,0.03328000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,64,0,1,fp8,fp8,0,0.0764160007238388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,64,128,1,float16,float16,0,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,64,128,1,float16,fp8,0,0.03321066747109095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,64,128,1,fp8,fp8,0,0.07713066538174947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,64,0,1,float16,fp8,0,0.03299200038115183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,64,0,1,fp8,fp8,0,0.07770133515199025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,64,128,1,float16,float16,0,0.03383466601371765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,64,0,1,float16,float16,0,0.03391999999682108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,64,128,1,float16,fp8,0,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,64,128,1,fp8,fp8,0,0.07813333471616109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,64,128,1,fp8,fp8,0,0.047040000557899475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,64,0,1,float16,fp8,0,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,64,0,1,fp8,fp8,0,0.07747200131416321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,64,128,1,float16,float16,0,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,64,0,1,float16,float16,0,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,64,128,1,float16,fp8,0,0.022709332406520844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,64,0,1,float16,fp8,0,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,64,0,1,fp8,fp8,0,0.04750399788220724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,64,128,1,float16,float16,0,0.021509334444999695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,64,0,1,float16,float16,0,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,64,128,1,float16,fp8,0,0.02184533327817917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,64,128,1,fp8,fp8,0,0.04670399924119314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,64,128,1,fp8,fp8,0,0.046538665890693665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,64,0,1,float16,fp8,0,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,64,0,1,float16,fp8,0,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,64,0,1,fp8,fp8,0,0.04641599953174591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,64,128,1,float16,float16,0,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,64,0,1,float16,float16,0,0.021920000513394673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,64,128,1,float16,fp8,0,0.021685334543387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,64,0,1,fp8,fp8,0,0.04633066554864248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,64,0,1,fp8,fp8,0,0.04719999929269155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,64,128,1,float16,float16,0,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,64,128,1,float16,float16,0,0.02204799900452296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,64,0,1,float16,float16,0,0.02197866638501485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,64,128,1,fp8,fp8,0,0.04688533147176107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,64,128,1,float16,fp8,0,0.022309333086013794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,64,0,1,fp8,fp8,0,0.04654933512210846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,64,128,1,fp8,fp8,0,0.04638933142026266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,64,0,1,float16,fp8,0,0.02221333235502243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,64,0,1,float16,float16,0,0.022298666338125866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,64,128,1,float16,fp8,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,64,0,1,float16,fp8,0,0.022463999688625336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,64,128,1,float16,float16,0,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,64,0,1,float16,float16,0,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,64,128,1,float16,fp8,0,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,64,128,1,fp8,fp8,0,0.030917334059874218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,64,0,1,float16,fp8,0,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,64,0,1,fp8,fp8,0,0.030991998811562855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,64,128,1,float16,float16,0,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,64,0,1,float16,float16,0,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,64,128,1,float16,fp8,0,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,64,0,1,float16,float16,0,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,64,128,1,fp8,fp8,0,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,64,0,1,float16,fp8,0,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,64,0,1,fp8,fp8,0,0.029839999973773956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,64,128,1,float16,float16,0,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,64,128,1,float16,fp8,0,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,64,128,1,fp8,fp8,0,0.029872000217437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,64,0,1,float16,fp8,0,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,64,0,1,fp8,fp8,0,0.029717333614826202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,64,128,1,float16,float16,0,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,64,0,1,float16,float16,0,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,64,128,1,float16,fp8,0,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,64,128,1,fp8,fp8,0,0.030293333033720653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,64,0,1,float16,fp8,0,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,64,0,1,fp8,fp8,0,0.03073599934577942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,64,128,1,float16,float16,0,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,64,0,1,float16,float16,0,0.015583999454975128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,64,128,1,float16,fp8,0,0.01573333392540614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,64,128,1,fp8,fp8,0,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,64,0,1,float16,fp8,0,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,64,0,1,fp8,fp8,0,0.03057066599527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,64,128,1,float16,float16,0,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,64,0,1,float16,float16,0,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,64,128,1,float16,fp8,0,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,64,128,1,fp8,fp8,0,0.022682666778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,64,0,1,float16,fp8,0,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,64,0,1,fp8,fp8,0,0.02269333352645238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,64,128,1,float16,float16,0,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,64,0,1,fp8,fp8,0,0.02201066662867864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,64,0,1,float16,float16,0,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,64,128,1,float16,fp8,0,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,64,128,1,fp8,fp8,0,0.022416000564893086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,64,128,1,fp8,fp8,0,0.022437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,64,0,1,float16,fp8,0,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,64,128,1,float16,float16,0,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,64,0,1,float16,float16,0,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,64,128,1,float16,fp8,0,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,64,0,1,float16,fp8,0,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,64,0,1,fp8,fp8,0,0.022437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,64,128,1,float16,float16,0,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,64,0,1,float16,float16,0,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,64,128,1,float16,fp8,0,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,64,128,1,fp8,fp8,0,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,64,0,1,float16,fp8,0,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,64,0,1,fp8,fp8,0,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,64,128,1,float16,float16,0,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,64,0,1,float16,float16,0,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,64,128,1,float16,fp8,0,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,64,128,1,fp8,fp8,0,0.022698665658632915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,64,0,1,float16,fp8,0,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,64,0,1,fp8,fp8,0,0.02272533377011617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,64,128,1,float16,float16,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,64,0,1,float16,float16,0,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,64,128,1,float16,fp8,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,64,128,1,fp8,fp8,0,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,64,0,1,float16,fp8,0,0.01202133297920227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,64,0,1,fp8,fp8,0,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,64,128,1,float16,float16,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,64,0,1,float16,float16,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,64,128,1,float16,fp8,0,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,64,128,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,64,0,1,float16,fp8,0,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,64,0,1,fp8,fp8,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,64,128,1,float16,float16,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,64,0,1,float16,float16,0,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,64,128,1,float16,fp8,0,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,64,128,1,fp8,fp8,0,0.01815466706951459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,64,0,1,float16,fp8,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,64,0,1,fp8,fp8,0,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,64,128,1,float16,float16,0,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,64,0,1,float16,float16,0,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,64,128,1,float16,fp8,0,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,64,128,1,fp8,fp8,0,0.018016000588734944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,64,0,1,float16,fp8,0,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,64,0,1,fp8,fp8,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,64,128,1,float16,float16,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,64,0,1,float16,float16,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,64,128,1,float16,fp8,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,64,128,1,fp8,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,64,0,1,float16,fp8,0,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,64,0,1,fp8,fp8,0,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,64,128,1,float16,float16,0,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,64,0,1,float16,float16,0,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,64,128,1,float16,fp8,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,64,128,1,fp8,fp8,0,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,64,0,1,float16,fp8,0,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,64,0,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,64,128,1,float16,float16,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,64,0,1,float16,float16,0,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,64,128,1,float16,fp8,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,64,128,1,fp8,fp8,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,64,0,1,float16,fp8,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,64,0,1,fp8,fp8,0,0.018016000588734944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,64,128,1,float16,float16,0,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,64,0,1,float16,float16,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,64,128,1,float16,fp8,0,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,64,128,1,fp8,fp8,0,0.017701332767804463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,64,0,1,float16,fp8,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,64,0,1,fp8,fp8,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,64,128,1,float16,float16,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,64,128,1,float16,fp8,0,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,64,128,1,fp8,fp8,0,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,64,0,1,float16,fp8,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,64,0,1,fp8,fp8,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,64,128,1,float16,float16,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,64,0,1,float16,float16,0,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,64,128,1,float16,fp8,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,64,128,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,64,0,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,64,0,1,fp8,fp8,0,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,64,128,1,float16,float16,0,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,64,0,1,float16,float16,0,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,64,128,1,float16,fp8,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,64,128,1,fp8,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,64,0,1,float16,fp8,0,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,64,0,1,fp8,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,64,128,1,float16,float16,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,64,0,1,float16,float16,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,64,128,1,float16,fp8,0,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,64,128,1,fp8,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,64,0,1,float16,fp8,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,64,0,1,fp8,fp8,0,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,64,128,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,64,0,1,float16,float16,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,64,128,1,float16,fp8,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,64,128,1,fp8,fp8,0,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,64,0,1,float16,fp8,0,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,64,0,1,fp8,fp8,0,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,64,128,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,64,0,1,float16,float16,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,64,128,1,float16,fp8,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,64,128,1,fp8,fp8,0,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,64,0,1,fp8,fp8,0,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,64,128,1,float16,float16,0,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,64,0,1,float16,float16,0,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,64,128,1,float16,fp8,0,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,64,128,1,fp8,fp8,0,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,64,0,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,64,0,1,fp8,fp8,0,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,64,128,1,float16,fp8,0,0.07667733232180278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,64,128,1,float16,float16,0,0.07670400043328603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,64,0,1,float16,float16,0,0.07691733539104462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,64,0,1,float16,fp8,0,0.0767626663049062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,64,128,1,fp8,fp8,0,0.2301386594772339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,64,0,1,fp8,fp8,0,0.2302560011545817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,64,128,1,float16,float16,0,0.0768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,64,0,1,float16,float16,0,0.07717866698900859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,64,128,1,float16,fp8,0,0.07717866698900859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,64,0,1,float16,fp8,0,0.07658666869004567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,64,128,1,float16,fp8,0,0.07774933179219563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,64,128,1,fp8,fp8,0,0.23100799322128296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,64,128,1,float16,float16,0,0.07807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,64,0,1,fp8,fp8,0,0.23028800884882608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,64,0,1,float16,float16,0,0.07796800136566162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,64,128,1,fp8,fp8,0,0.23149333397547403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,64,0,1,float16,fp8,0,0.07760000228881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,64,128,1,float16,float16,0,0.07985066870848338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,64,0,1,fp8,fp8,0,0.23121599356333414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,64,0,1,float16,float16,0,0.0800906668106715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,64,128,1,float16,fp8,0,0.0793333351612091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,64,0,1,float16,fp8,0,0.07981866598129272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,64,128,1,fp8,fp8,0,0.2341973384221395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,64,128,1,float16,float16,0,0.04619733492533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,64,0,1,float16,float16,0,0.046336000164349876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,64,0,1,fp8,fp8,0,0.23463465770085654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,64,128,1,float16,fp8,0,0.045237332582473755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,64,128,1,fp8,fp8,0,0.13082133730252585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,64,0,1,float16,fp8,0,0.045007998744646706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,64,0,1,fp8,fp8,0,0.13040533661842346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,64,128,1,float16,float16,0,0.04298133154710134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,64,0,1,float16,float16,0,0.042821332812309265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,64,128,1,float16,fp8,0,0.043050666650136314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,64,128,1,fp8,fp8,0,0.12376532951990764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,64,128,1,fp8,fp8,0,0.122789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,64,0,1,float16,fp8,0,0.04293866455554962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,64,0,1,fp8,fp8,0,0.12310933073361714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,64,128,1,float16,float16,0,0.04312000175317129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,64,0,1,float16,float16,0,0.04320000112056732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,64,128,1,float16,fp8,0,0.04331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,64,0,1,float16,fp8,0,0.043061330914497375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,64,0,1,fp8,fp8,0,0.12389333049456279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,64,128,1,float16,float16,0,0.04287466903527578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,64,0,1,float16,float16,0,0.04330133398373922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,64,128,1,float16,fp8,0,0.04331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,64,128,1,fp8,fp8,0,0.12326400478680928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,64,0,1,float16,fp8,0,0.042965332667032875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,64,0,1,fp8,fp8,0,0.12307733297348022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,64,128,1,fp8,fp8,0,0.12354666988054912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,64,128,1,float16,float16,0,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,64,0,1,float16,float16,0,0.04374399781227112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,64,128,1,float16,fp8,0,0.043562665581703186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,64,0,1,float16,fp8,0,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,64,128,1,float16,float16,0,0.026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,64,0,1,float16,float16,0,0.026447998980681103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,64,0,1,fp8,fp8,0,0.12357866764068604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,64,128,1,float16,float16,0,0.02630399912595749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,64,128,1,float16,fp8,0,0.026389333109060924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,64,0,1,float16,float16,0,0.02629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,64,128,1,fp8,fp8,0,0.07147733370463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,64,0,1,float16,fp8,0,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,64,0,1,fp8,fp8,0,0.07056533296902974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,64,128,1,float16,fp8,0,0.02641066660483678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,64,128,1,fp8,fp8,0,0.07053333520889282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,64,0,1,float16,fp8,0,0.026485333840052288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,64,0,1,fp8,fp8,0,0.06981866558392842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,64,128,1,float16,float16,0,0.02628266563018163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,64,0,1,float16,float16,0,0.02608533451954524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,64,128,1,float16,fp8,0,0.02626666675011317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,64,128,1,fp8,fp8,0,0.06991466879844666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,64,0,1,float16,fp8,0,0.026517334083716076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,64,0,1,fp8,fp8,0,0.06927466889222463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,64,128,1,float16,float16,0,0.026874666412671406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,64,0,1,float16,float16,0,0.02672533442576726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,64,128,1,float16,fp8,0,0.026928000152111053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,64,128,1,float16,float16,0,0.027322667340437572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,64,128,1,fp8,fp8,0,0.07063466807206471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,64,0,1,float16,fp8,0,0.02699200063943863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,64,0,1,fp8,fp8,0,0.07123733560244243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,64,0,1,float16,float16,0,0.02697066714366277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,64,128,1,float16,float16,0,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,64,128,1,float16,fp8,0,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,64,128,1,fp8,fp8,0,0.07017600039641063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,64,0,1,float16,fp8,0,0.027434666951497395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,64,0,1,fp8,fp8,0,0.07018666466077168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,64,0,1,float16,float16,0,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,64,128,1,float16,fp8,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,64,128,1,fp8,fp8,0,0.043247997760772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,64,0,1,float16,fp8,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,64,0,1,fp8,fp8,0,0.043562665581703186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,64,128,1,float16,float16,0,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,64,0,1,float16,float16,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,64,128,1,float16,fp8,0,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,64,0,1,float16,float16,0,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,64,128,1,fp8,fp8,0,0.04221866528193156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,64,0,1,float16,fp8,0,0.018426666657129925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,64,128,1,float16,float16,0,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,64,0,1,fp8,fp8,0,0.042463997999827065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,64,128,1,float16,float16,0,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,64,128,1,float16,fp8,0,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,64,128,1,fp8,fp8,0,0.0431573341290156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,64,0,1,float16,fp8,0,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,64,0,1,fp8,fp8,0,0.04293333490689596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,64,0,1,fp8,fp8,0,0.043061330914497375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,64,0,1,float16,float16,0,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,64,128,1,float16,fp8,0,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,64,128,1,float16,float16,0,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,64,128,1,fp8,fp8,0,0.04274666806062063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,64,0,1,float16,fp8,0,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,64,0,1,float16,float16,0,0.01860800012946129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,64,128,1,float16,fp8,0,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,64,128,1,fp8,fp8,0,0.042506664991378784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,64,0,1,float16,fp8,0,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,64,0,1,fp8,fp8,0,0.04261333247025808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,64,128,1,float16,float16,0,0.013829333086808523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,64,128,1,float16,fp8,0,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,64,0,1,float16,float16,0,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,64,128,1,fp8,fp8,0,0.029066666960716248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,64,0,1,float16,fp8,0,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,64,0,1,fp8,fp8,0,0.02874133239189784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,64,128,1,float16,float16,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,64,0,1,float16,float16,0,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,64,128,1,float16,fp8,0,0.01422400027513504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,64,128,1,fp8,fp8,0,0.028181334336598713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,64,0,1,float16,fp8,0,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,64,0,1,fp8,fp8,0,0.028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,64,128,1,float16,float16,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,64,0,1,float16,float16,0,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,64,0,1,float16,float16,0,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,64,128,1,float16,fp8,0,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,64,128,1,float16,fp8,0,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,64,128,1,fp8,fp8,0,0.028778667251269024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,64,0,1,float16,fp8,0,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,64,0,1,fp8,fp8,0,0.028538666665554047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,64,128,1,float16,float16,0,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,64,128,1,fp8,fp8,0,0.028143999477227528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,64,0,1,float16,fp8,0,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,64,0,1,fp8,fp8,0,0.029066666960716248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,64,128,1,float16,float16,0,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,64,0,1,float16,float16,0,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,64,128,1,float16,fp8,0,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,64,128,1,fp8,fp8,0,0.02880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,64,128,1,fp8,fp8,0,0.022287999590237934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,64,0,1,float16,fp8,0,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,64,0,1,fp8,fp8,0,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,64,128,1,float16,float16,0,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,64,0,1,float16,float16,0,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,64,128,1,float16,fp8,0,0.012442667037248611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,64,128,1,float16,fp8,0,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,64,0,1,float16,fp8,0,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,64,0,1,fp8,fp8,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,64,128,1,float16,float16,0,0.01246400053302447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,64,0,1,float16,float16,0,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,64,128,1,fp8,fp8,0,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,64,0,1,float16,fp8,0,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,64,0,1,fp8,fp8,0,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,64,128,1,float16,float16,0,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,64,0,1,float16,float16,0,0.012335999558369318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,64,128,1,float16,float16,0,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,64,128,1,float16,fp8,0,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,64,128,1,fp8,fp8,0,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,64,0,1,float16,fp8,0,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,64,0,1,fp8,fp8,0,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,64,0,1,float16,float16,0,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,64,128,1,float16,fp8,0,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,64,128,1,fp8,fp8,0,0.02178666740655899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,64,0,1,float16,fp8,0,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,64,0,1,fp8,fp8,0,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,64,128,1,float16,float16,0,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,64,0,1,fp8,fp8,0,0.021568000316619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,64,0,1,float16,float16,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,64,128,1,float16,fp8,0,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,64,128,1,fp8,fp8,0,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,64,0,1,float16,fp8,0,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,64,128,1,float16,float16,0,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,64,0,1,float16,float16,0,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,64,128,1,float16,fp8,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,64,128,1,fp8,fp8,0,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,64,0,1,fp8,fp8,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,64,128,1,float16,float16,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,64,0,1,float16,float16,0,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,64,128,1,float16,fp8,0,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,64,0,1,float16,float16,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,64,128,1,fp8,fp8,0,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,64,0,1,float16,fp8,0,0.01202133297920227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,64,0,1,fp8,fp8,0,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,64,128,1,float16,float16,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,64,128,1,float16,fp8,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,64,0,1,float16,float16,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,64,128,1,fp8,fp8,0,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,64,128,1,fp8,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,64,0,1,float16,fp8,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,64,0,1,fp8,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,64,0,1,fp8,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,64,128,1,float16,float16,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,64,128,1,float16,fp8,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,64,0,1,float16,fp8,0,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,64,128,1,float16,float16,0,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,64,128,1,float16,float16,0,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,64,0,1,float16,float16,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,64,128,1,float16,fp8,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,64,128,1,fp8,fp8,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,64,0,1,float16,fp8,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,64,0,1,fp8,fp8,0,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,64,0,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,64,0,1,float16,float16,0,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,64,128,1,float16,fp8,0,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,64,128,1,float16,fp8,0,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,64,128,1,fp8,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,64,0,1,float16,fp8,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,64,0,1,fp8,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,64,128,1,float16,float16,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,64,128,1,fp8,fp8,0,0.017893332988023758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,64,0,1,float16,fp8,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,64,0,1,fp8,fp8,0,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,64,128,1,float16,float16,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,64,0,1,float16,float16,0,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,64,128,1,float16,fp8,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,64,128,1,fp8,fp8,0,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,64,128,1,fp8,fp8,0,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,64,0,1,float16,fp8,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,64,0,1,fp8,fp8,0,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,64,128,1,float16,float16,0,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,64,128,1,float16,fp8,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,64,0,1,float16,fp8,0,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,64,0,1,fp8,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,64,128,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,64,0,1,float16,float16,0,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,64,128,1,float16,fp8,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,64,128,1,fp8,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,64,0,1,float16,fp8,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,64,0,1,fp8,fp8,0,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,64,128,1,float16,float16,0,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,64,0,1,float16,float16,0,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,64,128,1,float16,fp8,0,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,64,128,1,fp8,fp8,0,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,64,0,1,float16,fp8,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,64,0,1,fp8,fp8,0,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,64,128,1,float16,float16,0,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,64,0,1,float16,float16,0,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,64,128,1,float16,fp8,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,64,128,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,64,0,1,float16,fp8,0,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,64,0,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,64,128,1,float16,float16,0,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,64,0,1,float16,float16,0,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,64,128,1,float16,fp8,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,64,128,1,fp8,fp8,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,64,0,1,float16,fp8,0,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,64,0,1,fp8,fp8,0,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,64,128,1,float16,float16,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,64,0,1,float16,float16,0,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,64,128,1,float16,fp8,0,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,64,128,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,64,0,1,float16,fp8,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,64,0,1,fp8,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,64,128,1,float16,float16,0,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,64,0,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,64,128,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,64,128,1,fp8,fp8,0,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,64,0,1,float16,fp8,0,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,64,0,1,fp8,fp8,0,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,64,128,1,float16,float16,0,2.2094720204671225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,64,128,1,float16,fp8,0,2.1880213419596353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,64,128,1,fp8,fp8,0,2.7972532908121743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,64,128,1,float16,float16,0,2.3034507433573403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,64,128,1,float16,fp8,0,2.2874879837036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,64,128,1,fp8,fp8,0,2.879546801249186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,64,128,1,float16,float16,0,2.322085380554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,64,128,1,float16,fp8,0,2.302272001902262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,64,0,1,fp8,fp8,0,12.85141372680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,64,0,1,float16,float16,0,14.244181315104166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,64,0,1,float16,fp8,0,14.471380869547525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,64,128,1,fp8,fp8,0,2.898261388142904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,64,0,1,float16,float16,0,14.301872253417969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,64,128,1,float16,float16,0,2.3692426681518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,64,0,1,float16,fp8,0,14.26092274983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,64,0,1,fp8,fp8,0,12.898836771647135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,64,128,1,float16,fp8,0,2.3337546984354653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,64,0,1,float16,float16,0,14.267103830973307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,64,128,1,fp8,fp8,0,2.940570513407389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,64,128,1,float16,float16,0,1.3013280232747395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,64,128,1,float16,fp8,0,1.2795039812723796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,64,128,1,fp8,fp8,0,1.6228853861490886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,64,0,1,fp8,fp8,0,12.876714070638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,64,0,1,float16,fp8,0,14.3199462890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,64,128,1,float16,float16,0,1.0869332949320476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,64,0,1,float16,float16,0,7.335285186767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,64,0,1,float16,float16,0,14.269818623860678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,64,128,1,float16,fp8,0,1.0838932991027832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,64,0,1,float16,fp8,0,7.585744222005208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,64,0,1,fp8,fp8,0,6.68998908996582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,64,128,1,fp8,fp8,0,1.4237279891967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,64,0,1,fp8,fp8,0,13.014053344726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,64,128,1,float16,float16,0,1.1501333713531494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,64,128,1,float16,fp8,0,1.1389919916788738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,64,0,1,float16,fp8,0,15.781845092773438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,64,128,1,fp8,fp8,0,1.4716480573018391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,64,128,1,float16,float16,0,1.209818681081136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,64,0,1,float16,float16,0,8.097018559773764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,64,128,1,float16,fp8,0,1.155898650487264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,64,0,1,fp8,fp8,0,6.736501057942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,64,0,1,float16,fp8,0,7.073392232259114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,64,128,1,fp8,fp8,0,1.493578592936198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,64,0,1,float16,float16,0,7.319925308227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,64,0,1,fp8,fp8,0,6.532138824462891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,64,128,1,float16,float16,0,1.192464033762614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,64,0,1,float16,fp8,0,7.512351989746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,64,128,1,float16,fp8,0,1.1790080070495605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,64,128,1,fp8,fp8,0,1.5139306386311848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,64,0,1,float16,float16,0,7.860330581665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,64,128,1,float16,float16,0,0.6516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,64,128,1,float16,fp8,0,0.6591573158899943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,64,128,1,fp8,fp8,0,0.8373440106709799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,64,0,1,fp8,fp8,0,6.549936294555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,64,0,1,float16,fp8,0,7.459306716918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,64,128,1,float16,float16,0,0.5915199915568033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,64,0,1,float16,float16,0,3.74674129486084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,64,0,1,float16,float16,0,7.336687723795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,64,128,1,float16,fp8,0,0.5804479916890463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,64,0,1,fp8,fp8,0,3.402928034464518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,64,0,1,float16,fp8,0,3.7084051767985025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,64,128,1,fp8,fp8,0,0.7757493654886881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,64,0,1,fp8,fp8,0,6.585418701171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,64,0,1,float16,fp8,0,7.788602828979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,64,128,1,float16,float16,0,0.59061332543691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,64,128,1,float16,fp8,0,0.6010560194651285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,64,128,1,fp8,fp8,0,0.7637013594309489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,64,0,1,float16,float16,0,3.7018826802571616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,64,128,1,float16,float16,0,0.6124106645584106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,64,128,1,float16,fp8,0,0.6151253382364908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,64,0,1,fp8,fp8,0,3.2971359888712564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,64,0,1,float16,fp8,0,3.7282241185506186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,64,128,1,fp8,fp8,0,0.7748479843139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,64,0,1,float16,float16,0,3.671349207560221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,64,128,1,float16,float16,0,0.6637173493703207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,64,0,1,fp8,fp8,0,3.3015734354654946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,64,0,1,float16,fp8,0,3.8560053507486978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,64,128,1,float16,fp8,0,0.6064000129699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,64,0,1,float16,float16,0,3.717349370320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,64,128,1,fp8,fp8,0,0.8148266474405924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,64,128,1,float16,float16,0,0.42935999234517414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,64,128,1,float16,fp8,0,0.42803200085957843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,64,0,1,float16,fp8,0,3.695898691813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,64,0,1,fp8,fp8,0,3.291034698486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,64,0,1,float16,float16,0,3.7435146967569985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,64,128,1,fp8,fp8,0,0.5128000179926554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,64,0,1,float16,float16,0,2.0335520108540854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,64,0,1,fp8,fp8,0,3.3341598510742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,64,128,1,float16,float16,0,0.4251306851704915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,64,0,1,float16,fp8,0,3.8106613159179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,64,128,1,float16,fp8,0,0.42186133066813153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,64,0,1,fp8,fp8,0,1.7247999509175618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,64,0,1,float16,fp8,0,1.9799787203470867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,64,128,1,fp8,fp8,0,0.5076160033543905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,64,128,1,float16,float16,0,0.42264533042907715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,64,0,1,float16,float16,0,1.983839988708496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,64,128,1,float16,fp8,0,0.4215039809544881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,64,128,1,fp8,fp8,0,0.5080373287200928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,64,0,1,float16,fp8,0,1.9806079864501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,64,128,1,float16,float16,0,0.4213706652323405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,64,0,1,fp8,fp8,0,1.742458661397298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,64,0,1,float16,float16,0,1.981317361195882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,64,0,1,float16,fp8,0,2.0080906550089517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,64,0,1,fp8,fp8,0,1.7415092786153157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,64,128,1,float16,fp8,0,0.42340266704559326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,64,128,1,fp8,fp8,0,0.5094346602757772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,64,128,1,float16,float16,0,0.42719467480977374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,64,0,1,float16,float16,0,1.9665066401163738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,64,128,1,float16,fp8,0,0.4225279887517293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,64,128,1,fp8,fp8,0,0.5090133349100748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,64,0,1,fp8,fp8,0,1.7238133748372395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,64,0,1,float16,fp8,0,1.9800480206807454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,64,0,1,float16,float16,0,2.016554673512777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,64,0,1,float16,fp8,0,2.0061119397481284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,64,0,1,fp8,fp8,0,1.7341279983520508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,64,128,1,float16,float16,0,1.6471786499023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,64,128,1,float16,fp8,0,1.6167999903361003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,64,128,1,fp8,fp8,0,2.097210725148519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,64,128,1,float16,fp8,0,1.7169706026713054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,64,128,1,float16,float16,0,1.751237392425537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,64,128,1,fp8,fp8,0,2.1389333407084146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,64,128,1,float16,float16,0,1.7701279322306316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,64,128,1,float16,fp8,0,1.728816032409668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,64,0,1,float16,float16,0,8.281333287556967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,64,0,1,fp8,fp8,0,7.519423802693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,64,0,1,float16,fp8,0,8.340400060017904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,64,0,1,float16,float16,0,8.357338587443033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,64,128,1,fp8,fp8,0,2.183909257253011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,64,0,1,float16,fp8,0,8.360298792521158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,64,0,1,fp8,fp8,0,7.585626602172852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,64,128,1,float16,float16,0,1.7864960034688313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,64,128,1,float16,fp8,0,1.7647520701090496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,64,0,1,float16,float16,0,8.37884775797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,64,128,1,fp8,fp8,0,2.2142346700032554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,64,128,1,float16,float16,0,0.9739360014597574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,64,128,1,float16,fp8,0,0.9440800348917643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,64,128,1,fp8,fp8,0,1.2243253389994304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,64,0,1,fp8,fp8,0,7.603781382242839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,64,0,1,float16,fp8,0,8.328325271606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,64,128,1,float16,float16,0,0.8204800287882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,64,0,1,float16,float16,0,4.282896041870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,64,0,1,float16,float16,0,8.652751922607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,64,128,1,float16,fp8,0,0.8074773152669271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,64,0,1,fp8,fp8,0,7.648256301879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,64,128,1,fp8,fp8,0,1.0515999794006348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,64,0,1,float16,fp8,0,8.38211758931478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,64,0,1,float16,fp8,0,4.431722640991211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,64,0,1,fp8,fp8,0,3.960591952006022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,64,128,1,float16,float16,0,0.8263680140177408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,64,128,1,float16,fp8,0,0.8271893660227457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,64,0,1,float16,float16,0,4.35749880472819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,64,128,1,fp8,fp8,0,1.0867573420206706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,64,128,1,float16,float16,0,0.8432373205820719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,64,128,1,float16,fp8,0,0.8349386850992838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,64,0,1,fp8,fp8,0,3.795167922973633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,64,0,1,float16,fp8,0,4.439813296000163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,64,128,1,fp8,fp8,0,1.0978079636891682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,64,0,1,float16,float16,0,4.455578804016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,64,0,1,fp8,fp8,0,3.8090454737345376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,64,0,1,float16,fp8,0,4.238656044006348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,64,128,1,float16,float16,0,0.8872213363647461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,64,128,1,float16,fp8,0,0.8793599605560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,64,128,1,fp8,fp8,0,1.118890682856242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,64,0,1,float16,float16,0,4.367530822753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,64,128,1,float16,float16,0,0.5016479889551798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,64,128,1,float16,fp8,0,0.5103573401769003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,64,0,1,fp8,fp8,0,3.8321173985799155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,64,128,1,fp8,fp8,0,0.6440159877141317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,64,0,1,float16,fp8,0,4.393360137939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,64,0,1,float16,float16,0,2.1823946634928384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,64,0,1,float16,float16,0,4.384037335713704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,64,128,1,float16,float16,0,0.4519519805908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,64,0,1,fp8,fp8,0,3.8632373809814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,64,0,1,float16,fp8,0,4.46610673268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,64,128,1,float16,fp8,0,0.44522666931152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,64,0,1,float16,fp8,0,2.2169973055521646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,64,128,1,fp8,fp8,0,0.5907253424326578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,64,0,1,fp8,fp8,0,2.0348426500956216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,64,128,1,float16,float16,0,0.4697920083999634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,64,0,1,float16,float16,0,2.1453065872192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,64,128,1,float16,fp8,0,0.44893332322438556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,64,128,1,fp8,fp8,0,0.5825173457463583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,64,0,1,float16,fp8,0,2.1402880350748696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,64,128,1,float16,float16,0,0.4601813157399495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,64,0,1,fp8,fp8,0,1.949680010477702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,64,0,1,float16,float16,0,2.149397373199463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,64,128,1,float16,fp8,0,0.45334935188293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,64,0,1,float16,fp8,0,2.2590880393981934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,64,128,1,fp8,fp8,0,0.5837279955546061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,64,0,1,fp8,fp8,0,1.9686932563781738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,64,0,1,float16,float16,0,2.1516000429789224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,64,128,1,float16,float16,0,0.46194668610890705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,64,128,1,float16,fp8,0,0.4580053488413493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,64,128,1,fp8,fp8,0,0.5962239901224772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,64,128,1,float16,float16,0,0.32342400153477985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,64,0,1,fp8,fp8,0,1.9482666651407878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,64,0,1,float16,fp8,0,2.3057759602864585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,64,128,1,float16,fp8,0,0.32441065708796185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,64,128,1,fp8,fp8,0,0.3917173147201538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,64,0,1,float16,float16,0,2.2700799306233725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,64,0,1,float16,float16,0,1.1968106428782146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,64,0,1,fp8,fp8,0,1.9807146390279133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,64,0,1,float16,fp8,0,2.1946986516316733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,64,0,1,float16,fp8,0,1.2245279947916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,64,128,1,float16,float16,0,0.3213226596514384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,64,128,1,float16,fp8,0,0.3253920078277588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,64,0,1,fp8,fp8,0,1.0634079774220784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,64,128,1,fp8,fp8,0,0.3893226782480876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,64,128,1,float16,float16,0,0.3209493358929952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,64,0,1,float16,float16,0,1.1982613404591878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,64,128,1,float16,fp8,0,0.3258826732635498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,64,128,1,fp8,fp8,0,0.3901706536610921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,64,0,1,fp8,fp8,0,1.0475306510925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,64,0,1,float16,fp8,0,1.2181013425191243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,64,0,1,float16,float16,0,1.2105973561604817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,64,128,1,float16,float16,0,0.3250826597213745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,64,128,1,float16,fp8,0,0.3231839934984843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,64,0,1,float16,fp8,0,1.19050137201945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,64,128,1,fp8,fp8,0,0.39635733763376874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,64,0,1,fp8,fp8,0,1.0463039875030518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,64,128,1,float16,float16,0,0.3224053382873535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,64,0,1,float16,float16,0,1.2164053122202556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,64,128,1,float16,fp8,0,0.3258453408877055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,64,0,1,float16,fp8,0,1.2166186968485515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,64,0,1,fp8,fp8,0,1.0480533440907795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,64,128,1,fp8,fp8,0,0.39137065410614014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,64,0,1,float16,float16,0,1.220906654993693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,64,0,1,float16,fp8,0,1.1987626552581787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,64,0,1,fp8,fp8,0,1.0625706513722737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,64,128,1,float16,float16,0,1.3763306935628254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,64,128,1,float16,fp8,0,1.3446346918741863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,64,128,1,fp8,fp8,0,1.7459519704182942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,64,128,1,float16,float16,0,1.4415040016174316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,64,128,1,float16,fp8,0,1.431946595509847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,64,128,1,fp8,fp8,0,1.8158879280090332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,64,128,1,float16,float16,0,1.4517812728881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,64,0,1,float16,float16,0,5.963328043619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,64,0,1,fp8,fp8,0,5.375386555989583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,64,0,1,float16,fp8,0,5.934991836547852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,64,128,1,float16,fp8,0,1.4378827412923176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,64,0,1,float16,float16,0,5.937797546386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,64,128,1,fp8,fp8,0,1.826298713684082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,64,0,1,float16,fp8,0,6.167807896931966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,64,0,1,fp8,fp8,0,5.436176300048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,64,128,1,float16,float16,0,1.482591946919759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,64,128,1,float16,fp8,0,1.4688746134440105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,64,0,1,float16,float16,0,6.154624303181966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,64,128,1,fp8,fp8,0,1.8906720479329426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,64,128,1,float16,float16,0,0.8031040032704672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,64,128,1,float16,fp8,0,0.7837599913279215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,64,0,1,fp8,fp8,0,5.456495920817058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,64,128,1,fp8,fp8,0,1.0156373182932537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,64,0,1,float16,fp8,0,6.006117502848308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,64,0,1,float16,float16,0,3.1669333775838218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,64,128,1,float16,float16,0,0.6839893658955892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,64,0,1,float16,float16,0,6.24565315246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,64,0,1,fp8,fp8,0,5.4798933664957685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,64,128,1,float16,fp8,0,0.6781546274820963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,64,128,1,fp8,fp8,0,0.8849919637044271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,64,0,1,float16,fp8,0,3.1416587829589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,64,0,1,float16,fp8,0,6.251669565836589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,64,0,1,fp8,fp8,0,2.83568541208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,64,128,1,float16,float16,0,0.6965546607971191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,64,128,1,float16,fp8,0,0.7083679835001627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,64,0,1,float16,float16,0,3.124688148498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,64,128,1,fp8,fp8,0,0.9010720252990723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,64,0,1,float16,fp8,0,2.9816001256306968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,64,128,1,float16,float16,0,0.7147359848022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,64,0,1,fp8,fp8,0,2.708319981892904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,64,128,1,float16,fp8,0,0.691215991973877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,64,0,1,float16,float16,0,3.0505758921305337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,64,128,1,fp8,fp8,0,0.9117600123087565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,64,0,1,fp8,fp8,0,2.7313172022501626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,64,128,1,float16,float16,0,0.7142079671223959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,64,0,1,float16,fp8,0,3.0661118825276694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,64,128,1,float16,fp8,0,0.7088053226470947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,64,0,1,float16,float16,0,3.025045394897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,64,128,1,fp8,fp8,0,0.9325760205586752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,64,128,1,float16,float16,0,0.4120906591415405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,64,0,1,fp8,fp8,0,2.7256959279378257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,64,0,1,float16,fp8,0,2.965888023376465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,64,128,1,float16,fp8,0,0.41484800974527997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,64,128,1,fp8,fp8,0,0.5299466848373413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,64,0,1,float16,float16,0,1.5991039276123047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,64,0,1,float16,float16,0,3.055312156677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,64,128,1,float16,float16,0,0.37796799341837567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,64,0,1,fp8,fp8,0,2.7656853993733725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,64,0,1,float16,fp8,0,3.0008586247762046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,64,128,1,float16,fp8,0,0.3821013371149699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,64,0,1,float16,fp8,0,1.574885368347168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,64,128,1,fp8,fp8,0,0.48556800683339435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,64,0,1,fp8,fp8,0,1.4948800404866536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,64,128,1,float16,float16,0,0.3890773455301921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,64,128,1,float16,fp8,0,0.3768800099690755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,64,0,1,float16,float16,0,1.5719839731852214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,64,128,1,fp8,fp8,0,0.499728004137675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,64,0,1,float16,fp8,0,1.568544069925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,64,128,1,float16,float16,0,0.3914453188578288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,64,0,1,fp8,fp8,0,1.4634613990783691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,64,0,1,float16,float16,0,1.6244160334269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,64,128,1,float16,fp8,0,0.3884586493174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,64,128,1,fp8,fp8,0,0.5039199988047282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,64,0,1,float16,fp8,0,1.5433600743611653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,64,0,1,fp8,fp8,0,1.4189866383870442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,64,0,1,float16,float16,0,1.564245382944743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,64,128,1,float16,float16,0,0.3980213403701782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,64,128,1,float16,fp8,0,0.3884426752726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,64,128,1,fp8,fp8,0,0.5003466606140137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,64,0,1,float16,fp8,0,1.5741492907206218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,64,0,1,fp8,fp8,0,1.4550666809082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,64,128,1,float16,float16,0,0.2742080092430115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,64,0,1,float16,float16,0,1.5820800463358562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,64,128,1,float16,fp8,0,0.2737813393274943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,64,128,1,fp8,fp8,0,0.33718399206797284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,64,0,1,float16,float16,0,0.8906986713409424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,64,0,1,fp8,fp8,0,1.4631519317626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,64,0,1,float16,fp8,0,1.5560107231140137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,64,128,1,float16,float16,0,0.272490660349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,64,0,1,float16,fp8,0,0.8784746328989664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,64,128,1,float16,fp8,0,0.2751413385073344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,64,0,1,fp8,fp8,0,0.7765119870503744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,64,128,1,fp8,fp8,0,0.3338506619135539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,64,128,1,float16,float16,0,0.27536000808080036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,64,0,1,float16,float16,0,0.8784533341725668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,64,128,1,float16,fp8,0,0.2762613296508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,64,0,1,fp8,fp8,0,0.7642560005187988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,64,0,1,float16,fp8,0,0.8813599745432535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,64,128,1,fp8,fp8,0,0.33398934205373126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,64,0,1,float16,float16,0,0.890058676401774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,64,128,1,float16,float16,0,0.2763040065765381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,64,0,1,float16,fp8,0,0.8913280169169108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,64,0,1,fp8,fp8,0,0.764677365620931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,64,128,1,float16,fp8,0,0.2744906743367513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,64,128,1,fp8,fp8,0,0.33409066994984943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,64,0,1,float16,float16,0,0.8786880175272623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,64,128,1,float16,float16,0,0.2754826744397481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,64,128,1,float16,fp8,0,0.2743946711222331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,64,0,1,float16,fp8,0,0.8879093329111735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,64,0,1,fp8,fp8,0,0.7661279837290446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,64,128,1,fp8,fp8,0,0.3343786795934041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,64,0,1,float16,float16,0,0.880074659983317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,64,0,1,fp8,fp8,0,0.7772800127665201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,64,0,1,float16,fp8,0,0.8923679987589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,64,128,1,float16,float16,0,2.187503973642985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,64,128,1,float16,fp8,0,2.165605386098226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,64,128,1,fp8,fp8,0,2.7709919611612954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,64,128,1,float16,float16,0,2.2883520126342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,64,128,1,float16,fp8,0,2.2610079447428384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,64,128,1,fp8,fp8,0,2.853306770324707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,64,128,1,float16,float16,0,2.2966453234354653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,64,0,1,fp8,fp8,0,7.0969492594401045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,64,0,1,float16,float16,0,7.76962153116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,64,0,1,float16,fp8,0,7.849962870279948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,64,0,1,float16,float16,0,7.947717030843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,64,128,1,float16,fp8,0,2.2706185976664224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,64,0,1,fp8,fp8,0,7.207093556722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,64,128,1,fp8,fp8,0,2.8437118530273438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,64,0,1,float16,fp8,0,7.977455774943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,64,128,1,float16,float16,0,2.334719975789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,64,128,1,float16,fp8,0,2.3203412691752114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,64,0,1,float16,float16,0,8.00705083211263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,64,128,1,fp8,fp8,0,2.9006932576497397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,64,128,1,float16,float16,0,1.2878080209096272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,64,0,1,fp8,fp8,0,7.203802744547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,64,128,1,float16,fp8,0,1.2461120287577312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,64,0,1,float16,fp8,0,8.121599833170572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,64,128,1,fp8,fp8,0,1.6024319330851238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,64,128,1,float16,float16,0,1.0670133431752522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,64,0,1,float16,float16,0,4.254639943440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,64,0,1,float16,float16,0,7.981568018595378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,64,0,1,fp8,fp8,0,7.245802561442058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,64,128,1,float16,fp8,0,1.0537813504536946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,64,0,1,float16,fp8,0,7.969530741373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,64,0,1,fp8,fp8,0,3.770048141479492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,64,0,1,float16,fp8,0,4.086618741353353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,64,128,1,fp8,fp8,0,1.4224747021993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,64,128,1,float16,float16,0,1.1259360313415527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,64,0,1,float16,float16,0,4.238341331481934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,64,128,1,float16,fp8,0,1.112378676732381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,64,128,1,fp8,fp8,0,1.4791199366251628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,64,128,1,float16,float16,0,1.1970826784769695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,64,0,1,float16,fp8,0,3.9037707646687827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,64,0,1,fp8,fp8,0,3.549584070841471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,64,128,1,float16,fp8,0,1.1526880264282227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,64,0,1,float16,float16,0,4.013989448547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,64,0,1,float16,fp8,0,4.1942399342854815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,64,0,1,fp8,fp8,0,3.617504119873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,64,128,1,fp8,fp8,0,1.5022026697794597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,64,0,1,float16,float16,0,4.052554766337077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,64,128,1,float16,float16,0,1.1718133290608723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,64,128,1,float16,fp8,0,1.1450133323669434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,64,128,1,fp8,fp8,0,1.5151306788126628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,64,128,1,float16,float16,0,0.6298506657282511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,64,0,1,fp8,fp8,0,3.65230401357015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,64,0,1,float16,fp8,0,4.115359942118327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,64,128,1,float16,fp8,0,0.6144959926605225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,64,128,1,fp8,fp8,0,0.8039360046386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,64,0,1,float16,float16,0,4.135306676228841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,64,0,1,float16,float16,0,2.0610292752583823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,64,128,1,float16,float16,0,0.564192016919454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,64,0,1,fp8,fp8,0,3.683072090148926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,64,128,1,float16,fp8,0,0.5471466779708862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,64,0,1,float16,fp8,0,2.0425599416097007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,64,0,1,float16,fp8,0,3.9701385498046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,64,0,1,fp8,fp8,0,1.8876320521036785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,64,128,1,fp8,fp8,0,0.7306826909383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,64,0,1,float16,float16,0,2.0009066263834634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,64,128,1,float16,float16,0,0.5648159980773926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,64,128,1,float16,fp8,0,0.5425066550572714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,64,128,1,fp8,fp8,0,0.7374239762624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,64,0,1,fp8,fp8,0,1.8236319224039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,64,128,1,float16,float16,0,0.5554346640904745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,64,0,1,float16,fp8,0,2.130261262257894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,64,0,1,float16,float16,0,1.9882559776306152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,64,128,1,float16,fp8,0,0.557808001836141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,64,128,1,fp8,fp8,0,0.7202666600545248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,64,0,1,fp8,fp8,0,1.8778026898701985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,64,0,1,float16,fp8,0,2.0178027153015137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,64,128,1,float16,float16,0,0.5733333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,64,0,1,float16,float16,0,1.9973972638448079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,64,128,1,float16,fp8,0,0.559930682182312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,64,128,1,fp8,fp8,0,0.75709335009257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,64,128,1,float16,float16,0,0.3364373445510864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,64,0,1,fp8,fp8,0,1.8964319229125977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,64,0,1,float16,fp8,0,1.9868319829305012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,64,128,1,float16,fp8,0,0.3438346783320109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,64,0,1,float16,float16,0,2.003210703531901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,64,128,1,fp8,fp8,0,0.4342720111211141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,64,0,1,float16,float16,0,1.0595413049062092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,64,0,1,fp8,fp8,0,1.8466827074686687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,64,128,1,float16,float16,0,0.315013329188029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,64,0,1,float16,fp8,0,1.9949493408203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,64,0,1,fp8,fp8,0,0.9559306303660074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,64,0,1,float16,fp8,0,1.0685973167419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,64,128,1,float16,fp8,0,0.31153066953023273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,64,128,1,fp8,fp8,0,0.3935786485671997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,64,0,1,float16,float16,0,1.0456053415934246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,64,128,1,float16,float16,0,0.3093973398208618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,64,0,1,float16,fp8,0,1.0402560234069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,64,0,1,fp8,fp8,0,0.9386453628540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,64,128,1,float16,fp8,0,0.3072906732559204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,64,128,1,fp8,fp8,0,0.40801600615183514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,64,0,1,float16,float16,0,1.0450613498687744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,64,0,1,float16,fp8,0,1.0367786884307861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,64,128,1,float16,float16,0,0.31063467264175415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,64,0,1,fp8,fp8,0,0.9403626918792725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,64,128,1,fp8,fp8,0,0.4034453233083089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,64,128,1,float16,fp8,0,0.31497599681218463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,64,0,1,float16,float16,0,1.0462506612141926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,64,128,1,float16,float16,0,0.31243733565012616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,64,128,1,float16,fp8,0,0.3171893358230591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,64,0,1,float16,fp8,0,1.0625440279642742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,64,0,1,fp8,fp8,0,0.924394687016805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,64,128,1,fp8,fp8,0,0.4087146520614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,64,128,1,float16,float16,0,0.22501333554585776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,64,0,1,float16,float16,0,1.0729066530863445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,64,128,1,float16,fp8,0,0.2265546719233195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,64,128,1,fp8,fp8,0,0.27432000637054443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,64,0,1,float16,float16,0,0.5715786616007487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,64,0,1,float16,fp8,0,1.0680053234100342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,64,0,1,fp8,fp8,0,0.9289706548055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,64,0,1,float16,fp8,0,0.5688800017038981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,64,128,1,float16,float16,0,0.22404799858729044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,64,0,1,fp8,fp8,0,0.5323413213094076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,64,128,1,float16,fp8,0,0.2225653330485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,64,128,1,fp8,fp8,0,0.2728373408317566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,64,0,1,float16,float16,0,0.5755680004755656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,64,128,1,float16,float16,0,0.22329066197077432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,64,0,1,float16,fp8,0,0.5684959888458252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,64,0,1,float16,float16,0,0.5764053265253702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,64,0,1,fp8,fp8,0,0.528650681177775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,64,128,1,float16,fp8,0,0.22450133164723715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,64,128,1,fp8,fp8,0,0.2723093430201213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,64,128,1,float16,float16,0,0.22395733992258707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,64,0,1,float16,fp8,0,0.5697813431421915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,64,128,1,float16,fp8,0,0.2253226637840271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,64,0,1,fp8,fp8,0,0.5350613196690878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,64,128,1,fp8,fp8,0,0.27562133471171063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,64,0,1,float16,float16,0,0.5686399936676025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,64,128,1,float16,float16,0,0.22392000754674277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,64,0,1,float16,fp8,0,0.5741386810938517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,64,0,1,fp8,fp8,0,0.5274133284886678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,64,128,1,float16,fp8,0,0.2259733279546102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,64,128,1,fp8,fp8,0,0.27480000257492065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,64,0,1,float16,float16,0,0.5767840147018433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,64,0,1,float16,fp8,0,0.5701226790746053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,64,0,1,fp8,fp8,0,0.5274133284886678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,64,128,1,float16,float16,0,1.639296054840088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,64,128,1,float16,fp8,0,1.6106079419453938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,64,128,1,fp8,fp8,0,2.0738560358683267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,64,128,1,float16,float16,0,1.716826597849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,64,128,1,float16,fp8,0,1.699002742767334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,64,0,1,fp8,fp8,0,4.275530815124512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,64,0,1,float16,float16,0,4.915290514628093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,64,128,1,fp8,fp8,0,2.1481547355651855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,64,0,1,float16,fp8,0,4.735232035319011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,64,0,1,float16,float16,0,4.818688074747722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,64,128,1,float16,float16,0,1.7233279546101887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,64,128,1,float16,fp8,0,1.7050239245096843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,64,0,1,float16,fp8,0,4.856207847595215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,64,128,1,fp8,fp8,0,2.1629600524902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,64,0,1,fp8,fp8,0,4.326128005981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,64,128,1,float16,float16,0,1.7626454035441081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,64,128,1,float16,fp8,0,1.7428852717081706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,64,0,1,float16,float16,0,4.978581428527832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,64,128,1,float16,float16,0,0.9553439617156982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,64,128,1,fp8,fp8,0,2.179514726003011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,64,0,1,fp8,fp8,0,4.362112045288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,64,0,1,float16,fp8,0,4.86956787109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,64,128,1,float16,fp8,0,0.9321706295013428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,64,0,1,float16,float16,0,4.924288113911946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,64,0,1,float16,float16,0,2.522986729939779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,64,128,1,fp8,fp8,0,1.195797363917033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,64,128,1,float16,float16,0,0.7910559972127279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,64,128,1,float16,fp8,0,0.7969013055165609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,64,0,1,float16,fp8,0,2.456175963083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,64,0,1,float16,fp8,0,4.842613220214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,64,0,1,fp8,fp8,0,2.35699733098348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,64,0,1,fp8,fp8,0,4.390096028645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,64,128,1,fp8,fp8,0,1.0147093137105305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,64,128,1,float16,float16,0,0.8225119908650717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,64,0,1,float16,float16,0,2.3235839207967124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,64,128,1,float16,fp8,0,0.8135093053181967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,64,0,1,float16,fp8,0,2.3036160469055176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,64,0,1,fp8,fp8,0,2.175093332926432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,64,128,1,fp8,fp8,0,1.0498720010121663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,64,128,1,float16,float16,0,0.8180533250172933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,64,0,1,float16,float16,0,2.3346986770629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,64,128,1,float16,fp8,0,0.8332800070444742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,64,128,1,fp8,fp8,0,1.0744586785634358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,64,0,1,float16,fp8,0,2.326810677846273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,64,0,1,fp8,fp8,0,2.205413341522217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,64,128,1,float16,float16,0,0.8425652980804443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,64,0,1,float16,float16,0,2.374154726664225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,64,128,1,float16,fp8,0,0.8305866718292236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,64,0,1,fp8,fp8,0,2.176410675048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,64,128,1,fp8,fp8,0,1.1038080056508381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,64,0,1,float16,fp8,0,2.3890026410420737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,64,128,1,float16,float16,0,0.4772799809773763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,64,128,1,float16,fp8,0,0.4675626754760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,64,0,1,float16,float16,0,2.4296852747599282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,64,128,1,fp8,fp8,0,0.5981760025024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,64,0,1,float16,float16,0,1.2737973531087239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,64,0,1,float16,fp8,0,2.358821392059326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,64,128,1,float16,float16,0,0.41838399569193524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,64,0,1,fp8,fp8,0,2.1925172805786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,64,128,1,float16,fp8,0,0.4077599843343099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,64,0,1,float16,fp8,0,1.2685546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,64,0,1,fp8,fp8,0,1.1614186763763428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,64,128,1,fp8,fp8,0,0.5561333497365316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,64,128,1,float16,float16,0,0.4169280131657918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,64,0,1,float16,float16,0,1.1983306407928467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,64,128,1,float16,fp8,0,0.426202654838562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,64,128,1,fp8,fp8,0,0.5620693365732828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,64,0,1,fp8,fp8,0,1.0944373607635498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,64,0,1,float16,fp8,0,1.1946079730987549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,64,0,1,float16,float16,0,1.1915573279062908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,64,128,1,float16,float16,0,0.42659199237823486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,64,128,1,float16,fp8,0,0.432368000348409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,64,0,1,float16,fp8,0,1.218938668568929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,64,128,1,fp8,fp8,0,0.5476853450139364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,64,0,1,fp8,fp8,0,1.0933972994486492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,64,128,1,float16,float16,0,0.43436264991760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,64,0,1,float16,float16,0,1.2281813621520996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,64,128,1,float16,fp8,0,0.4395413398742676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,64,0,1,float16,fp8,0,1.225221316019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,64,0,1,fp8,fp8,0,1.1106826464335124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,64,128,1,fp8,fp8,0,0.5577600002288818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,64,128,1,float16,float16,0,0.26203733682632446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,64,128,1,float16,fp8,0,0.26369067033131915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,64,0,1,float16,float16,0,1.2420000235239665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,64,128,1,fp8,fp8,0,0.33268266916275024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,64,0,1,float16,float16,0,0.6729599634806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,64,0,1,float16,fp8,0,1.1965440114339192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,64,0,1,fp8,fp8,0,1.1457226276397705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,64,128,1,float16,float16,0,0.2429493268330892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,64,0,1,fp8,fp8,0,0.5991040070851644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,64,0,1,float16,fp8,0,0.6533013184865316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,64,128,1,float16,fp8,0,0.23905599117279053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,64,128,1,fp8,fp8,0,0.31390400727589923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,64,0,1,float16,float16,0,0.6559893290201823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,64,128,1,float16,float16,0,0.2432266672452291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,64,0,1,float16,fp8,0,0.6306133270263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,64,0,1,fp8,fp8,0,0.5740213394165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,64,128,1,float16,fp8,0,0.2344640096028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,64,128,1,fp8,fp8,0,0.31617067257563275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,64,0,1,float16,float16,0,0.6420426766077677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,64,0,1,float16,fp8,0,0.65229864915212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,64,128,1,float16,float16,0,0.2358186642328898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,64,0,1,fp8,fp8,0,0.5654613176981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,64,128,1,float16,fp8,0,0.23704532782236734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,64,0,1,float16,float16,0,0.657583991686503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,64,128,1,fp8,fp8,0,0.3171839912732442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,64,128,1,float16,float16,0,0.23931199312210083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,64,0,1,float16,fp8,0,0.6421066522598267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,64,0,1,fp8,fp8,0,0.5774186849594116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,64,128,1,float16,fp8,0,0.23985066016515097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,64,128,1,fp8,fp8,0,0.3225119908650716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,64,0,1,float16,float16,0,0.6493546565373739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,64,128,1,float16,float16,0,0.17391467094421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,64,128,1,float16,fp8,0,0.17409066359202066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,64,0,1,float16,float16,0,0.36529068152109784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,64,0,1,fp8,fp8,0,0.5622506539026896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,64,128,1,fp8,fp8,0,0.21902932723363241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,64,0,1,float16,fp8,0,0.6583413283030192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,64,0,1,float16,fp8,0,0.3614506721496582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,64,128,1,float16,float16,0,0.1717546582221985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,64,0,1,fp8,fp8,0,0.33455467224121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,64,0,1,float16,float16,0,0.3633120059967041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,64,128,1,float16,fp8,0,0.17431465784708658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,64,128,1,fp8,fp8,0,0.21554666757583618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,64,0,1,float16,fp8,0,0.3625813325246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,64,0,1,fp8,fp8,0,0.33340267340342206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,64,128,1,float16,float16,0,0.17420266071955362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,64,128,1,float16,fp8,0,0.17509865760803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,64,0,1,float16,fp8,0,0.3636480172475179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,64,0,1,float16,float16,0,0.3585386673609416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,64,128,1,fp8,fp8,0,0.21605332692464194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,64,0,1,fp8,fp8,0,0.3380906581878662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,64,128,1,float16,float16,0,0.17423999309539795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,64,128,1,float16,fp8,0,0.17566933234532675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,64,0,1,float16,float16,0,0.3609973192214966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,64,128,1,fp8,fp8,0,0.2162719964981079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,64,0,1,float16,fp8,0,0.36113067468007404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,64,0,1,fp8,fp8,0,0.3381066719690959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,64,128,1,float16,float16,0,0.17596266667048135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,64,0,1,float16,float16,0,0.36207465330759686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,64,128,1,float16,fp8,0,0.17510400215784708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,64,128,1,fp8,fp8,0,0.21405333280563354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,64,0,1,float16,fp8,0,0.3583093484242757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,64,0,1,fp8,fp8,0,0.33699198563893634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,64,128,1,float16,float16,0,2.171130657196045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,64,128,1,float16,fp8,0,2.153343995412191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,64,128,1,float16,float16,0,2.2657920519510903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,64,128,1,fp8,fp8,0,2.7400906880696616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,64,0,1,float16,float16,0,4.692293485005696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,64,0,1,fp8,fp8,0,4.278287887573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,64,128,1,float16,fp8,0,2.2450613975524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,64,0,1,float16,fp8,0,4.607669194539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,64,128,1,fp8,fp8,0,2.833183924357096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,64,0,1,float16,float16,0,4.7175947825113935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,64,128,1,float16,float16,0,2.274986743927002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,64,128,1,float16,fp8,0,2.2561705907185874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,64,0,1,float16,fp8,0,4.827626546223958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,64,128,1,fp8,fp8,0,2.849360148111979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,64,0,1,fp8,fp8,0,4.342565218607585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,64,0,1,float16,float16,0,4.946037292480469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,64,128,1,float16,float16,0,2.3242026964823403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,64,128,1,float16,fp8,0,2.3110507329305015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,64,0,1,float16,fp8,0,4.823461214701335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,64,128,1,fp8,fp8,0,2.888490676879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,64,128,1,float16,float16,0,1.273418664932251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,64,0,1,fp8,fp8,0,4.356693267822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,64,0,1,float16,float16,0,4.960245450337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,64,128,1,float16,fp8,0,1.2660746574401855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,64,128,1,fp8,fp8,0,1.579210599263509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,64,0,1,float16,fp8,0,4.770693461100261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,64,0,1,float16,float16,0,2.5804266929626465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,64,128,1,float16,float16,0,1.0649066766103108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,64,0,1,float16,fp8,0,2.479647954305013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,64,0,1,fp8,fp8,0,2.318549315134684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,64,0,1,fp8,fp8,0,4.4197438557942705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,64,128,1,float16,fp8,0,1.0174667040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,64,128,1,fp8,fp8,0,1.3647680282592773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,64,0,1,float16,float16,0,2.287029266357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,64,128,1,float16,float16,0,1.1136639912923176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,64,128,1,float16,fp8,0,1.1025599638621013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,64,0,1,float16,fp8,0,2.2674400011698403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,64,0,1,fp8,fp8,0,2.120570659637451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,64,128,1,fp8,fp8,0,1.4116746584574382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,64,0,1,float16,float16,0,2.416325410207113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,64,128,1,float16,float16,0,1.120901346206665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,64,0,1,float16,fp8,0,2.3366026878356934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,64,128,1,float16,fp8,0,1.1179146766662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,64,0,1,fp8,fp8,0,2.187013308207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,64,128,1,fp8,fp8,0,1.440773328145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,64,0,1,float16,float16,0,2.3561867078145347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,64,128,1,float16,float16,0,1.1528213024139404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,64,0,1,float16,fp8,0,2.351792017618815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,64,128,1,float16,fp8,0,1.1430559953053792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,64,0,1,fp8,fp8,0,2.2318666776021323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,64,128,1,fp8,fp8,0,1.4709280331929524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,64,128,1,float16,float16,0,0.63155198097229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,64,128,1,float16,fp8,0,0.610858678817749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,64,0,1,float16,float16,0,2.3945226669311523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,64,0,1,float16,float16,0,1.2656319936116536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,64,0,1,float16,fp8,0,2.3795040448506675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,64,128,1,fp8,fp8,0,0.7874560356140137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,64,0,1,fp8,fp8,0,2.2666239738464355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,64,128,1,float16,float16,0,0.5537013212839762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,64,0,1,float16,fp8,0,1.243226687113444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,64,0,1,fp8,fp8,0,1.1652692953745525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,64,128,1,float16,fp8,0,0.5431786775588989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,64,128,1,fp8,fp8,0,0.6891466776529948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,64,0,1,float16,float16,0,1.18340269724528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,64,128,1,float16,float16,0,0.5460746685663859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,64,0,1,float16,fp8,0,1.1709439754486084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,64,128,1,float16,fp8,0,0.5491146643956503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,64,0,1,fp8,fp8,0,1.1013600031534831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,64,128,1,fp8,fp8,0,0.696565310160319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,64,0,1,float16,float16,0,1.190826654434204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,64,128,1,float16,float16,0,0.5451306502024332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,64,0,1,float16,fp8,0,1.1804693539937336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,64,0,1,fp8,fp8,0,1.0790879726409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,64,128,1,float16,fp8,0,0.5476106802622477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,64,128,1,fp8,fp8,0,0.7044800122578939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,64,0,1,float16,float16,0,1.1938880284627278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,64,128,1,float16,float16,0,0.5556053320566813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,64,0,1,float16,fp8,0,1.190837303797404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,64,128,1,float16,fp8,0,0.5529919862747192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,64,0,1,fp8,fp8,0,1.0852800210316975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,64,128,1,fp8,fp8,0,0.7369919617970785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,64,0,1,float16,float16,0,1.1802720228830974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,64,128,1,float16,float16,0,0.32993600765864056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,64,128,1,float16,fp8,0,0.3211519916852315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,64,0,1,float16,float16,0,0.6444693406422933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,64,128,1,fp8,fp8,0,0.4206133286158244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,64,0,1,float16,fp8,0,1.1747946739196777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,64,0,1,fp8,fp8,0,1.0991466840108235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,64,0,1,float16,fp8,0,0.6416266759236654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,64,0,1,fp8,fp8,0,0.5952959855397543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,64,128,1,float16,float16,0,0.2879626750946045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,64,128,1,float16,fp8,0,0.2887466748555501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,64,0,1,float16,float16,0,0.6124746799468994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,64,128,1,fp8,fp8,0,0.3760319948196411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,64,128,1,float16,float16,0,0.2963306705156962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,64,0,1,fp8,fp8,0,0.549397349357605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,64,0,1,float16,fp8,0,0.6051573355992635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,64,128,1,fp8,fp8,0,0.3797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,64,128,1,float16,fp8,0,0.29125332832336426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,64,0,1,float16,float16,0,0.6128533283869425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,64,0,1,float16,fp8,0,0.6088053385416666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,64,128,1,float16,fp8,0,0.2943306763966878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,64,128,1,float16,float16,0,0.2954346736272176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,64,0,1,fp8,fp8,0,0.5540106693903605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,64,0,1,float16,float16,0,0.6145813465118408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,64,128,1,fp8,fp8,0,0.3855733474095662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,64,128,1,float16,float16,0,0.3027999997138977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,64,0,1,float16,fp8,0,0.6108426650365194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,64,0,1,fp8,fp8,0,0.5599466562271118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,64,128,1,float16,fp8,0,0.2990880012512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,64,128,1,fp8,fp8,0,0.38489067554473877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,64,0,1,float16,float16,0,0.6235466798146566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,64,128,1,float16,float16,0,0.1834133267402649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,64,0,1,float16,float16,0,0.32923199733098346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,64,0,1,fp8,fp8,0,0.5628213485081991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,64,0,1,float16,fp8,0,0.6229973236719767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,64,128,1,float16,fp8,0,0.18389866749445596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,64,128,1,fp8,fp8,0,0.23691733678181967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,64,0,1,float16,fp8,0,0.32920000950495404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,64,0,1,fp8,fp8,0,0.31018133958180744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,64,128,1,float16,float16,0,0.1681386629740397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,64,128,1,float16,fp8,0,0.16537066300710043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,64,0,1,float16,float16,0,0.3104213277498881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,64,128,1,fp8,fp8,0,0.2178666591644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,64,0,1,float16,fp8,0,0.31681599219640094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,64,128,1,float16,float16,0,0.16562133034070334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,64,0,1,fp8,fp8,0,0.29364800453186035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,64,128,1,float16,fp8,0,0.16657066345214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,64,0,1,float16,float16,0,0.3144479990005493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,64,128,1,fp8,fp8,0,0.2204213341077169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,64,0,1,float16,fp8,0,0.31062932809193927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,64,0,1,fp8,fp8,0,0.29478933413823444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,64,128,1,float16,float16,0,0.16870399316151938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,64,0,1,float16,float16,0,0.3144960006078084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,64,128,1,float16,fp8,0,0.16713599363962808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,64,128,1,fp8,fp8,0,0.22379199663798013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,64,0,1,float16,fp8,0,0.31457600990931195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,64,128,1,float16,float16,0,0.16801599661509195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,64,0,1,fp8,fp8,0,0.295904000600179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,64,128,1,float16,fp8,0,0.1713386575380961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,64,0,1,float16,float16,0,0.3177493413289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,64,128,1,fp8,fp8,0,0.22131200631459555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,64,0,1,float16,fp8,0,0.317194660504659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,64,0,1,fp8,fp8,0,0.30009599526723224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,64,128,1,float16,float16,0,0.12577066818873087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,64,0,1,float16,float16,0,0.19923200209935507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,64,128,1,float16,fp8,0,0.12568533420562744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,64,128,1,fp8,fp8,0,0.1572373310724894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,64,0,1,float16,fp8,0,0.19785600900650024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,64,0,1,fp8,fp8,0,0.18534932533899942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,64,128,1,float16,float16,0,0.12177067001660664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,64,0,1,float16,float16,0,0.19452800353368124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,64,128,1,float16,fp8,0,0.12123733758926392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,64,128,1,fp8,fp8,0,0.14544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,64,0,1,float16,fp8,0,0.19478933016459146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,64,0,1,fp8,fp8,0,0.18269866704940796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,64,128,1,float16,float16,0,0.12271466851234436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,64,0,1,float16,fp8,0,0.19708800315856934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,64,0,1,float16,float16,0,0.1951733430226644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,64,128,1,float16,fp8,0,0.12362666924794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,64,128,1,fp8,fp8,0,0.14916800459225973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,64,0,1,fp8,fp8,0,0.18465065956115723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,64,128,1,float16,float16,0,0.12203199664751689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,64,0,1,float16,float16,0,0.19539733727773032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,64,128,1,float16,fp8,0,0.12333866953849792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,64,128,1,fp8,fp8,0,0.15069866180419922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,64,0,1,float16,fp8,0,0.19643733898798624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,64,0,1,fp8,fp8,0,0.18437333901723227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,64,128,1,float16,float16,0,0.12388799587885539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,64,0,1,float16,float16,0,0.1961066722869873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,64,0,1,fp8,fp8,0,0.1852746605873108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,64,128,1,float16,fp8,0,0.12414933244387309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,64,128,1,fp8,fp8,0,0.15531733632087708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,64,0,1,float16,fp8,0,0.19892799854278564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,64,128,1,float16,float16,0,1.5846986770629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,64,128,1,float16,fp8,0,1.570101261138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,64,128,1,fp8,fp8,0,2.0268853505452475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,64,128,1,float16,float16,0,1.6411412556966145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,64,0,1,float16,float16,0,2.844357490539551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,64,0,1,fp8,fp8,0,2.6176533699035645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,64,0,1,float16,fp8,0,2.829029401143392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,64,128,1,float16,fp8,0,1.6254080136617024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,64,0,1,float16,float16,0,2.928448041280111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,64,128,1,fp8,fp8,0,2.062826633453369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,64,128,1,float16,float16,0,1.6568586031595867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,64,0,1,float16,fp8,0,2.887200037638346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,64,128,1,float16,fp8,0,1.641248067220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,64,0,1,fp8,fp8,0,2.659338633219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,64,0,1,float16,float16,0,2.918527921040853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,64,128,1,fp8,fp8,0,2.117952028910319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,64,128,1,float16,float16,0,1.712725321451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,64,128,1,float16,fp8,0,1.6852000554402669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,64,0,1,float16,fp8,0,2.899936040242513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,64,0,1,fp8,fp8,0,2.7096853256225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,64,128,1,fp8,fp8,0,2.1516745885213218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,64,128,1,float16,float16,0,0.9456746578216553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,64,0,1,float16,float16,0,2.9688905080159507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,64,128,1,float16,fp8,0,0.923151969909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,64,0,1,float16,fp8,0,2.945349375406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,64,128,1,fp8,fp8,0,1.1759040355682373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,64,0,1,float16,float16,0,1.5804212888081868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,64,0,1,fp8,fp8,0,2.76694393157959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,64,128,1,float16,float16,0,0.7747466564178467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,64,0,1,float16,fp8,0,1.5473119417826335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,64,0,1,fp8,fp8,0,1.4820799827575684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,64,128,1,float16,fp8,0,0.7663893699645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,64,0,1,float16,float16,0,1.4122506777445476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,64,128,1,fp8,fp8,0,0.998410701751709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,64,128,1,float16,float16,0,0.7924959659576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,64,0,1,float16,fp8,0,1.4155467351277669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,64,0,1,fp8,fp8,0,1.305829366048177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,64,128,1,float16,fp8,0,0.783888022104899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,64,0,1,float16,float16,0,1.4555360476175945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,64,128,1,fp8,fp8,0,1.020965337753296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,64,128,1,float16,float16,0,0.8032000064849854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,64,0,1,fp8,fp8,0,1.3463999430338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,64,0,1,float16,fp8,0,1.4191039403279622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,64,128,1,float16,fp8,0,0.7919360001881918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,64,128,1,fp8,fp8,0,1.047935962677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,64,0,1,float16,float16,0,1.4535999298095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,64,0,1,fp8,fp8,0,1.3287306626637776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,64,0,1,float16,fp8,0,1.4461013476053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,64,128,1,float16,float16,0,0.8297973473866781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,64,128,1,float16,fp8,0,0.8297653198242188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,64,128,1,fp8,fp8,0,1.0782399972279866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,64,128,1,float16,float16,0,0.467087984085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,64,0,1,float16,float16,0,1.478373368581136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,64,128,1,float16,fp8,0,0.46013331413269043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,64,0,1,float16,float16,0,0.7912639776865641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,64,0,1,float16,fp8,0,1.4502399762471516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,64,0,1,fp8,fp8,0,1.3802132606506348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,64,128,1,fp8,fp8,0,0.59826131661733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,64,0,1,float16,fp8,0,0.7810666561126709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,64,0,1,fp8,fp8,0,0.7222293217976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,64,128,1,float16,float16,0,0.40459199746449787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,64,0,1,float16,float16,0,0.7311573028564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,64,128,1,float16,fp8,0,0.399946649869283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,64,128,1,float16,float16,0,0.4137226740519206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,64,128,1,fp8,fp8,0,0.5267519950866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,64,0,1,fp8,fp8,0,0.6772106488545736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,64,0,1,float16,float16,0,0.7329813639322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,64,0,1,float16,fp8,0,0.7253066698710123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,64,128,1,float16,fp8,0,0.4034506479899089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,64,128,1,fp8,fp8,0,0.5307413339614868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,64,0,1,float16,fp8,0,0.7292479674021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,64,0,1,fp8,fp8,0,0.687013308207194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,64,128,1,float16,float16,0,0.4126346508661906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,64,128,1,float16,fp8,0,0.4089119831720988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,64,128,1,fp8,fp8,0,0.5369439919789633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,64,0,1,float16,float16,0,0.7377706368764242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,64,0,1,fp8,fp8,0,0.6828853289286295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,64,0,1,float16,fp8,0,0.7323413689931234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,64,128,1,float16,float16,0,0.4207199811935425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,64,128,1,float16,fp8,0,0.4172319968541463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,64,0,1,float16,float16,0,0.7537919680277506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,64,128,1,fp8,fp8,0,0.5500479936599731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,64,128,1,float16,float16,0,0.24820266167322794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,64,0,1,float16,fp8,0,0.7414133548736572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,64,128,1,float16,fp8,0,0.24634132782618204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,64,0,1,float16,float16,0,0.41546134154001874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,64,0,1,fp8,fp8,0,0.6924426555633545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,64,128,1,fp8,fp8,0,0.3171093265215556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,64,0,1,float16,fp8,0,0.41389866669972736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,64,128,1,float16,float16,0,0.22127999862035116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,64,0,1,fp8,fp8,0,0.3731360038121541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,64,128,1,float16,fp8,0,0.21851199865341187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,64,0,1,float16,float16,0,0.3904053370157878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,64,128,1,fp8,fp8,0,0.289792001247406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,64,0,1,float16,fp8,0,0.3873279889424642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,64,128,1,float16,float16,0,0.22207999229431152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,64,0,1,fp8,fp8,0,0.34809601306915283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,64,128,1,fp8,fp8,0,0.2919680078824361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,64,128,1,float16,fp8,0,0.21918932596842447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,64,0,1,float16,float16,0,0.3901120026906331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,64,0,1,float16,fp8,0,0.38842666149139404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,64,128,1,float16,float16,0,0.22472532590230307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,64,0,1,fp8,fp8,0,0.3494986693064372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,64,128,1,float16,fp8,0,0.2222399910291036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,64,0,1,float16,float16,0,0.39398399988810223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,64,128,1,fp8,fp8,0,0.2948746681213379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,64,0,1,float16,fp8,0,0.3917440176010132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,64,0,1,fp8,fp8,0,0.34999998410542804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,64,128,1,float16,float16,0,0.22897599140803018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,64,128,1,float16,fp8,0,0.2274506688117981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,64,0,1,float16,float16,0,0.39608001708984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,64,128,1,fp8,fp8,0,0.29969600836435956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,64,0,1,float16,fp8,0,0.39446401596069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,64,0,1,fp8,fp8,0,0.3544906775156657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,64,128,1,float16,float16,0,0.14154666662216187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,64,0,1,float16,float16,0,0.2160266637802124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,64,128,1,float16,fp8,0,0.14173866311709085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,64,128,1,fp8,fp8,0,0.18722132841746011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,64,0,1,float16,fp8,0,0.2170613408088684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,64,128,1,float16,fp8,0,0.12956266601880392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,64,0,1,fp8,fp8,0,0.2042400042215983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,64,128,1,float16,float16,0,0.12899200121561685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,64,0,1,float16,float16,0,0.20385066668192545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,64,128,1,fp8,fp8,0,0.17032533884048462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,64,0,1,float16,fp8,0,0.2026133338610331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,64,128,1,float16,float16,0,0.1302079955736796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,64,0,1,fp8,fp8,0,0.19099199771881104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,64,0,1,float16,float16,0,0.20249066750208536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,64,128,1,float16,fp8,0,0.12929600477218628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,64,128,1,fp8,fp8,0,0.17364267508188883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,64,0,1,float16,fp8,0,0.20284799734751383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,64,0,1,fp8,fp8,0,0.19212265809377035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,64,128,1,float16,float16,0,0.1313706636428833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,64,128,1,float16,fp8,0,0.13036800424257913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,64,0,1,float16,float16,0,0.20544532934824625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,64,128,1,fp8,fp8,0,0.17330666383107504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,64,0,1,float16,fp8,0,0.2039733330408732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,64,0,1,fp8,fp8,0,0.19426133235295615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,64,128,1,float16,float16,0,0.1334933340549469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,64,0,1,float16,float16,0,0.20627733071645102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,64,128,1,float16,fp8,0,0.13310399651527405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,64,128,1,fp8,fp8,0,0.17731734116872153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,64,0,1,float16,fp8,0,0.20681599775950113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,64,0,1,fp8,fp8,0,0.1962453325589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,64,128,1,float16,float16,0,0.10083199540774028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,64,0,1,float16,float16,0,0.13569600383440653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,64,128,1,float16,fp8,0,0.10004799564679463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,64,128,1,fp8,fp8,0,0.12453333536783855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,64,0,1,float16,fp8,0,0.13518933455149332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,64,0,1,fp8,fp8,0,0.12802666425704956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,64,128,1,float16,float16,0,0.0960586667060852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,64,0,1,float16,float16,0,0.13249599933624268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,64,128,1,float16,fp8,0,0.09637866417566936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,64,128,1,fp8,fp8,0,0.1142080028851827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,64,0,1,float16,fp8,0,0.132042666276296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,64,0,1,fp8,fp8,0,0.12475200494130452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,64,128,1,float16,float16,0,0.0962506632010142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,64,0,1,float16,float16,0,0.1320319970448812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,64,128,1,float16,fp8,0,0.09609066446622212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,64,128,1,fp8,fp8,0,0.1142080028851827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,64,0,1,float16,fp8,0,0.13271466890970865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,64,0,1,fp8,fp8,0,0.1253973344961802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,64,128,1,float16,float16,0,0.09627733627955119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,64,0,1,float16,float16,0,0.13211199641227722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,64,128,1,float16,fp8,0,0.09620267152786255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,64,128,1,fp8,fp8,0,0.11468266447385152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,64,0,1,float16,fp8,0,0.1327946682771047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,64,0,1,fp8,fp8,0,0.12519466876983643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,64,128,1,float16,float16,0,0.09789333740870158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,64,0,1,float16,float16,0,0.13379733761151633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,64,128,1,float16,fp8,0,0.09780266880989075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,64,128,1,fp8,fp8,0,0.11691199739774068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,64,0,1,float16,fp8,0,0.1328213314215342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,64,0,1,fp8,fp8,0,0.12660266955693564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,64,128,1,float16,float16,0,2.1113012631734214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,64,128,1,float16,fp8,0,2.0894452730814614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,64,128,1,fp8,fp8,0,2.6157120068868003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,64,128,1,float16,float16,0,2.1262453397115073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,64,0,1,float16,float16,0,3.036341349283854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,64,0,1,fp8,fp8,0,2.722240130106608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,64,0,1,float16,fp8,0,3.012144088745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,64,0,1,float16,float16,0,3.0486720403035483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,64,128,1,float16,fp8,0,2.0922133127848306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,64,128,1,float16,float16,0,2.153301397959391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,64,128,1,fp8,fp8,0,2.661381403605143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,64,0,1,fp8,fp8,0,2.7636801401774087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,64,0,1,float16,fp8,0,3.0177653630574546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,64,128,1,float16,fp8,0,2.1260533332824707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,64,0,1,float16,float16,0,3.071824073791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,64,128,1,fp8,fp8,0,2.682842572530111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,64,128,1,float16,float16,0,2.215504010518392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,64,0,1,fp8,fp8,0,2.7713867823282876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,64,128,1,float16,fp8,0,2.1800907452901206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,64,0,1,float16,fp8,0,3.0443414052327475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,64,0,1,float16,float16,0,3.1448373794555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,64,128,1,fp8,fp8,0,2.828762690226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,64,128,1,float16,float16,0,1.2361280123392742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,64,128,1,float16,fp8,0,1.2090293566385906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,64,0,1,float16,fp8,0,3.104703903198242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,64,0,1,float16,float16,0,1.702367941538493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,64,128,1,fp8,fp8,0,1.5707039833068848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,64,0,1,float16,fp8,0,1.695701281229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,64,0,1,fp8,fp8,0,2.90828800201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,64,128,1,float16,float16,0,1.023792028427124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,64,128,1,float16,fp8,0,1.0310133298238118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,64,0,1,fp8,fp8,0,1.5887680053710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,64,128,1,fp8,fp8,0,1.3070666790008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,64,0,1,float16,float16,0,1.4940800666809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,64,128,1,float16,float16,0,1.040992021560669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,64,0,1,float16,fp8,0,1.5031733512878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,64,0,1,fp8,fp8,0,1.3894826571146648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,64,128,1,float16,fp8,0,1.0574719905853271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,64,0,1,float16,float16,0,1.5054826736450195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,64,128,1,fp8,fp8,0,1.355797290802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,64,0,1,fp8,fp8,0,1.3873653411865234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,64,0,1,float16,fp8,0,1.497978687286377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,64,128,1,float16,float16,0,1.0537973244984944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,64,128,1,float16,fp8,0,1.063424030939738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,64,0,1,float16,float16,0,1.5306560198465984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,64,128,1,fp8,fp8,0,1.375168005625407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,64,0,1,float16,fp8,0,1.5138452847798665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,64,0,1,fp8,fp8,0,1.4177546501159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,64,128,1,float16,float16,0,1.1036960283915203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,64,128,1,float16,fp8,0,1.0758612950642903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,64,0,1,float16,float16,0,1.5931359926859539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,64,128,1,fp8,fp8,0,1.3987840016682942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,64,128,1,float16,float16,0,0.6188533306121826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,64,128,1,float16,fp8,0,0.6058559815088908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,64,0,1,float16,float16,0,0.8651306629180908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,64,0,1,float16,fp8,0,1.5519092877705891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,64,0,1,fp8,fp8,0,1.450602690378825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,64,128,1,fp8,fp8,0,0.7638506889343262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,64,0,1,float16,fp8,0,0.8540000120798746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,64,128,1,float16,float16,0,0.5163360039393107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,64,0,1,fp8,fp8,0,0.7922720114390055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,64,128,1,float16,fp8,0,0.5196373462677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,64,0,1,float16,float16,0,0.7535253365834554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,64,128,1,fp8,fp8,0,0.6740266482035319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,64,128,1,float16,float16,0,0.531770666440328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,64,0,1,float16,fp8,0,0.7488533655802408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,64,0,1,fp8,fp8,0,0.6863893667856852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,64,128,1,float16,fp8,0,0.5203839937845866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,64,0,1,float16,float16,0,0.7534933090209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,64,128,1,fp8,fp8,0,0.6910879611968994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,64,0,1,float16,fp8,0,0.7531466484069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,64,0,1,fp8,fp8,0,0.7025653521219889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,64,128,1,float16,float16,0,0.5322293440500895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,64,128,1,float16,fp8,0,0.5171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,64,0,1,float16,float16,0,0.7651999791463217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,64,128,1,fp8,fp8,0,0.6857173442840576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,64,0,1,float16,fp8,0,0.7531786759694418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,64,128,1,float16,float16,0,0.542085329691569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,64,0,1,fp8,fp8,0,0.7184586524963379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,64,128,1,float16,fp8,0,0.5392853418986002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,64,0,1,float16,float16,0,0.7712852954864502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,64,128,1,fp8,fp8,0,0.6984053452809652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,64,128,1,float16,float16,0,0.31780799229939777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,64,0,1,float16,fp8,0,0.7717013359069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,64,0,1,fp8,fp8,0,0.7245173454284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,64,128,1,float16,fp8,0,0.3152693311373393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,64,128,1,fp8,fp8,0,0.39976000785827637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,64,0,1,float16,float16,0,0.4429226716359456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,64,0,1,float16,fp8,0,0.4296853144963582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,64,0,1,fp8,fp8,0,0.3938773473103841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,64,128,1,float16,float16,0,0.27350399891535443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,64,128,1,fp8,fp8,0,0.3619146744410197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,64,128,1,float16,fp8,0,0.2704853415489197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,64,0,1,float16,float16,0,0.3964960177739461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,64,0,1,float16,fp8,0,0.39449600378672284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,64,0,1,fp8,fp8,0,0.3644533157348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,64,128,1,float16,float16,0,0.276528000831604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,64,0,1,float16,float16,0,0.398362676302592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,64,0,1,fp8,fp8,0,0.3649226824442546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,64,128,1,float16,fp8,0,0.2727466622988383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,64,128,1,fp8,fp8,0,0.3654346863428752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,64,0,1,float16,fp8,0,0.3936533530553182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,64,128,1,float16,float16,0,0.282154659430186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,64,128,1,float16,fp8,0,0.2760266661643982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,64,0,1,float16,float16,0,0.40859198570251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,64,128,1,fp8,fp8,0,0.3654186725616455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,64,0,1,float16,fp8,0,0.4034613370895386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,64,0,1,fp8,fp8,0,0.35974399248758954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,64,128,1,float16,float16,0,0.2872533400853475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,64,0,1,float16,float16,0,0.40721599260965985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,64,128,1,float16,fp8,0,0.28229333957036334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,64,128,1,fp8,fp8,0,0.3789866765340169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,64,0,1,float16,fp8,0,0.4036906560262044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,64,128,1,float16,float16,0,0.17375467220942178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,64,0,1,fp8,fp8,0,0.3744373321533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,64,0,1,float16,float16,0,0.23874133825302124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,64,128,1,float16,fp8,0,0.1733013391494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,64,128,1,fp8,fp8,0,0.2253119945526123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,64,0,1,float16,float16,0,0.20691200097401938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,64,0,1,float16,fp8,0,0.23625600337982178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,64,0,1,fp8,fp8,0,0.21104532480239868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,64,128,1,float16,float16,0,0.15310399731000265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,64,128,1,float16,fp8,0,0.15204800168673197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,64,128,1,fp8,fp8,0,0.20526933670043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,64,0,1,float16,float16,0,0.20905067523320517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,64,0,1,float16,fp8,0,0.20645334323247275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,64,0,1,fp8,fp8,0,0.1957333286603292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,64,128,1,float16,float16,0,0.15461333592732748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,64,128,1,float16,fp8,0,0.15149333079655966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,64,128,1,fp8,fp8,0,0.2063360015551249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,64,0,1,float16,fp8,0,0.20430932442347208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,64,0,1,fp8,fp8,0,0.19834667444229126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,64,128,1,float16,float16,0,0.1579253375530243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,64,0,1,float16,float16,0,0.20733867088953653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,64,128,1,float16,fp8,0,0.15545599659283957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,64,128,1,fp8,fp8,0,0.2071253259976705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,64,0,1,float16,fp8,0,0.20699199040730795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,64,0,1,fp8,fp8,0,0.19712533553441366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,64,128,1,float16,float16,0,0.16128533085187277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,64,0,1,float16,float16,0,0.21118400494257608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,64,128,1,float16,fp8,0,0.15966399510701498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,64,128,1,fp8,fp8,0,0.21115734179814658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,64,128,1,float16,fp8,0,0.10265599687894185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,64,0,1,float16,fp8,0,0.20928533871968588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,64,0,1,fp8,fp8,0,0.20120000839233398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,64,128,1,float16,float16,0,0.1009173293908437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,64,0,1,float16,float16,0,0.12495999534924825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,64,128,1,fp8,fp8,0,0.13567999998728433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,64,0,1,float16,fp8,0,0.12613866726557413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,64,0,1,fp8,fp8,0,0.12181333700815837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,64,128,1,float16,float16,0,0.0913866659005483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,64,0,1,float16,float16,0,0.11560533444086711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,64,128,1,float16,fp8,0,0.09148266911506653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,64,128,1,fp8,fp8,0,0.11609599987665813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,64,0,1,float16,float16,0,0.11530133088429768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,64,0,1,float16,fp8,0,0.11519466837247212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,64,0,1,fp8,fp8,0,0.10940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,64,128,1,float16,float16,0,0.09232532978057861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,64,128,1,float16,fp8,0,0.09190932909647624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,64,128,1,fp8,fp8,0,0.12150399883588155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,64,0,1,float16,fp8,0,0.11529599626859029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,64,0,1,fp8,fp8,0,0.10895466804504395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,64,128,1,float16,float16,0,0.09238933523495992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,64,0,1,float16,float16,0,0.11710932850837708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,64,0,1,float16,float16,0,0.11805333693822224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,64,128,1,float16,fp8,0,0.09290132919947307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,64,128,1,fp8,fp8,0,0.12537599603335062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,64,0,1,float16,fp8,0,0.1169599990049998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,64,0,1,fp8,fp8,0,0.11466667056083679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,64,0,1,fp8,fp8,0,0.11219200491905212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,64,128,1,float16,float16,0,0.09537600477536519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,64,128,1,float16,fp8,0,0.09524266918500264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,64,128,1,fp8,fp8,0,0.12870933612187704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,64,0,1,float16,fp8,0,0.11867733796437581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,64,0,1,fp8,fp8,0,0.07991466422875722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,64,128,1,float16,float16,0,0.07300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,64,0,1,float16,float16,0,0.08474666873613994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,64,128,1,float16,fp8,0,0.07213333249092102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,64,128,1,fp8,fp8,0,0.09131733576456706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,64,0,1,float16,fp8,0,0.08342933654785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,64,128,1,float16,float16,0,0.06939200063546498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,64,0,1,float16,float16,0,0.08135466774304707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,64,128,1,float16,fp8,0,0.06952000161012013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,64,128,1,fp8,fp8,0,0.08725333213806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,64,0,1,float16,fp8,0,0.08266133566697438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,64,0,1,fp8,fp8,0,0.07828799883524577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,64,0,1,fp8,fp8,0,0.07893333335717519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,64,128,1,float16,float16,0,0.06938666601975758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,64,128,1,float16,float16,0,0.07019199927647908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,64,0,1,float16,float16,0,0.08159466584523518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,64,128,1,float16,fp8,0,0.06941866874694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,64,128,1,fp8,fp8,0,0.08460799853006999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,64,0,1,float16,fp8,0,0.08218133449554443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,64,0,1,fp8,fp8,0,0.07797866563002269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,64,0,1,float16,float16,0,0.08187733093897502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,64,128,1,float16,fp8,0,0.06958400209744771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,64,128,1,fp8,fp8,0,0.08517332871754964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,64,0,1,float16,fp8,0,0.08283199866612752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,64,128,1,float16,float16,0,0.06942399839560191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,64,0,1,float16,float16,0,0.08148266871770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,64,128,1,float16,fp8,0,0.06970133384068807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,64,128,1,fp8,fp8,0,0.08682666222254436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,64,0,1,float16,fp8,0,0.08212799827257793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,64,0,1,fp8,fp8,0,0.0784853349129359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,64,128,1,float16,float16,0,1.5725280443827312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,64,128,1,float16,fp8,0,1.5476640065511067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,64,0,1,float16,float16,0,1.9910400708516438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,64,128,1,fp8,fp8,0,1.9374133745829265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,64,0,1,float16,fp8,0,1.9720800717671711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,64,128,1,float16,float16,0,1.5615253448486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,64,0,1,fp8,fp8,0,1.7694560686747234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,64,128,1,float16,fp8,0,1.5441439946492512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,64,0,1,float16,float16,0,1.9955946604410808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,64,128,1,fp8,fp8,0,1.9784533182779949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,64,128,1,float16,float16,0,1.59062925974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,64,0,1,float16,fp8,0,1.9648906389872234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,64,0,1,fp8,fp8,0,1.8164693514506023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,64,0,1,float16,float16,0,2.0101706186930337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,64,128,1,float16,fp8,0,1.565285364786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,64,128,1,fp8,fp8,0,2.013066609700521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,64,0,1,float16,fp8,0,1.990026632944743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,64,128,1,float16,float16,0,1.6412053108215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,64,0,1,fp8,fp8,0,1.8427519798278809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,64,0,1,float16,float16,0,2.0643253326416016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,64,128,1,float16,fp8,0,1.611120065053304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,64,128,1,fp8,fp8,0,2.051248073577881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,64,128,1,float16,float16,0,0.919871966044108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,64,128,1,float16,fp8,0,0.9002400239308676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,64,0,1,float16,float16,0,1.1345333258310955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,64,0,1,float16,fp8,0,2.0411094029744468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,64,128,1,fp8,fp8,0,1.1310559908548992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,64,0,1,fp8,fp8,0,1.88372802734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,64,0,1,float16,fp8,0,1.1098933219909668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,64,128,1,float16,float16,0,0.7705173492431641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,64,0,1,fp8,fp8,0,1.0414613087972004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,64,128,1,float16,fp8,0,0.7552213668823242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,64,0,1,float16,float16,0,0.9778186480204264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,64,128,1,fp8,fp8,0,0.9776853720347086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,64,0,1,float16,fp8,0,0.9663840134938558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,64,0,1,fp8,fp8,0,0.8874826431274414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,64,128,1,float16,float16,0,0.7747039794921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,64,128,1,float16,fp8,0,0.7691306273142496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,64,0,1,float16,float16,0,0.9968106746673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,64,128,1,fp8,fp8,0,0.9990453720092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,64,0,1,float16,fp8,0,0.9757973353068033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,64,128,1,float16,float16,0,0.790992021560669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,64,0,1,fp8,fp8,0,0.8994399706522623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,64,128,1,float16,fp8,0,0.790175994237264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,64,0,1,float16,float16,0,1.0006293455759685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,64,128,1,fp8,fp8,0,1.0069013436635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,64,0,1,float16,fp8,0,1.001253366470337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,64,0,1,fp8,fp8,0,0.9201813538869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,64,128,1,float16,float16,0,0.8187893231709799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,64,128,1,float16,fp8,0,0.8027573426564535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,64,0,1,float16,float16,0,1.0421120325724285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,64,128,1,float16,float16,0,0.4673653443654378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,64,128,1,fp8,fp8,0,1.0494186878204346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,64,0,1,float16,fp8,0,1.0189173221588135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,64,128,1,float16,fp8,0,0.4539946715037028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,64,0,1,fp8,fp8,0,0.9460426966349283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,64,0,1,float16,float16,0,0.5759733517964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,64,128,1,fp8,fp8,0,0.5727466742197672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,64,0,1,fp8,fp8,0,0.5254773298899332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,64,0,1,float16,fp8,0,0.568506677945455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,64,128,1,float16,float16,0,0.39818131923675537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,64,0,1,float16,float16,0,0.5085279941558838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,64,128,1,float16,fp8,0,0.38415467739105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,64,128,1,fp8,fp8,0,0.5092853307723999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,64,0,1,float16,fp8,0,0.5021493434906006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,64,0,1,fp8,fp8,0,0.45795198281606037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,64,128,1,float16,float16,0,0.39906664689381915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,64,0,1,float16,float16,0,0.5148799816767374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,64,128,1,float16,fp8,0,0.3933546543121338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,64,128,1,fp8,fp8,0,0.5238240162531534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,64,0,1,float16,fp8,0,0.49797332286834717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,64,0,1,fp8,fp8,0,0.4737173318862915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,64,128,1,float16,float16,0,0.4061013460159302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,64,128,1,float16,fp8,0,0.3992160161336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,64,0,1,float16,float16,0,0.5084480047225952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,64,128,1,fp8,fp8,0,0.521343986193339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,64,0,1,float16,fp8,0,0.5081706841786703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,64,0,1,fp8,fp8,0,0.47793598969777423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,64,128,1,float16,float16,0,0.40722668170928955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,64,0,1,float16,float16,0,0.5256373484929403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,64,128,1,float16,fp8,0,0.4118986527125041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,64,128,1,float16,float16,0,0.24404799938201904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,64,128,1,fp8,fp8,0,0.5346826712290446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,64,0,1,float16,fp8,0,0.5136266549428304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,64,0,1,fp8,fp8,0,0.4829759995142619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,64,0,1,float16,float16,0,0.30323733886082965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,64,128,1,float16,fp8,0,0.23985066016515097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,64,128,1,fp8,fp8,0,0.3107999960581462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,64,0,1,float16,fp8,0,0.29840532938639325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,64,0,1,fp8,fp8,0,0.27112533648808795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,64,0,1,float16,fp8,0,0.26410667101542157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,64,128,1,float16,float16,0,0.2116373380025228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,64,0,1,float16,float16,0,0.2667520046234131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,64,128,1,float16,fp8,0,0.20875734090805054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,64,128,1,fp8,fp8,0,0.2828106681505839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,64,0,1,fp8,fp8,0,0.24465600649515787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,64,128,1,float16,float16,0,0.2137599984804789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,64,0,1,float16,float16,0,0.27003733317057294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,64,128,1,float16,fp8,0,0.2087786595026652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,64,128,1,fp8,fp8,0,0.28097599744796753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,64,0,1,float16,fp8,0,0.26467732588450116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,64,0,1,fp8,fp8,0,0.2432533303896586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,64,128,1,float16,float16,0,0.21900266408920288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,64,0,1,float16,float16,0,0.27296000719070435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,64,128,1,float16,fp8,0,0.21461333831151327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,64,128,1,float16,float16,0,0.21955732504526773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,64,128,1,fp8,fp8,0,0.2821013331413269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,64,0,1,float16,fp8,0,0.2709386746088664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,64,0,1,fp8,fp8,0,0.24409600098927817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,64,0,1,float16,float16,0,0.27820799748102826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,64,128,1,float16,fp8,0,0.220143993695577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,64,128,1,float16,float16,0,0.13507200280825296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,64,128,1,fp8,fp8,0,0.28914666175842285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,64,0,1,float16,fp8,0,0.27428799867630005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,64,0,1,fp8,fp8,0,0.25036799907684326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,64,0,1,float16,float16,0,0.15945067008336386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,64,0,1,fp8,fp8,0,0.14869333306948343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,64,128,1,float16,fp8,0,0.13550933202107748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,64,128,1,fp8,fp8,0,0.17547732591629028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,64,0,1,float16,fp8,0,0.15974400440851846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,64,128,1,float16,float16,0,0.11897599697113037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,64,0,1,float16,float16,0,0.13904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,64,128,1,float16,fp8,0,0.11780266960461934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,64,128,1,fp8,fp8,0,0.1601759990056356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,64,0,1,float16,fp8,0,0.1399679978688558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,64,0,1,fp8,fp8,0,0.13366400202115378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,64,128,1,float16,float16,0,0.11979732910792033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,64,0,1,float16,float16,0,0.14017066359519958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,64,128,1,float16,fp8,0,0.11800000071525574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,64,128,1,fp8,fp8,0,0.16011733810106912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,64,0,1,float16,fp8,0,0.13966400424639383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,64,0,1,fp8,fp8,0,0.13498133420944214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,64,128,1,float16,float16,0,0.12148800492286682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,64,128,1,fp8,fp8,0,0.16351466377576193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,64,0,1,float16,float16,0,0.14127467075983682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,64,128,1,float16,fp8,0,0.12016533811887105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,64,0,1,float16,fp8,0,0.14018133282661438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,64,0,1,fp8,fp8,0,0.13685866196950278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,64,128,1,fp8,fp8,0,0.16683199008305868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,64,128,1,float16,float16,0,0.12591466307640076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,64,0,1,float16,float16,0,0.1442506710688273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,64,128,1,float16,fp8,0,0.12425067027409871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,64,0,1,float16,fp8,0,0.14421866337458292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,64,128,1,fp8,fp8,0,0.11002133289972942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,64,0,1,fp8,fp8,0,0.13970133662223816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,64,128,1,float16,float16,0,0.08004266520341237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,64,0,1,float16,float16,0,0.08945600191752116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,64,128,1,float16,fp8,0,0.08201066652933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,64,0,1,float16,fp8,0,0.08904000123341878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,64,0,1,fp8,fp8,0,0.08799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,64,128,1,float16,float16,0,0.07172800103823344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,64,0,1,float16,float16,0,0.08207466701666515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,64,128,1,float16,fp8,0,0.07231999933719635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,64,128,1,fp8,fp8,0,0.09317866961161296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,64,0,1,float16,fp8,0,0.08346666892369588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,64,0,1,fp8,fp8,0,0.07759466767311096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,64,0,1,float16,fp8,0,0.08323733508586884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,64,128,1,float16,float16,0,0.07205333312352498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,64,0,1,float16,float16,0,0.08338666955629985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,64,128,1,float16,fp8,0,0.0721013347307841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,64,128,1,fp8,fp8,0,0.09221333265304565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,64,0,1,fp8,fp8,0,0.07930666704972585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,64,128,1,float16,float16,0,0.0721973329782486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,64,0,1,float16,float16,0,0.08403733372688293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,64,128,1,float16,fp8,0,0.07253866891066234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,64,128,1,fp8,fp8,0,0.09430399537086487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,64,0,1,float16,fp8,0,0.0830506682395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,64,0,1,fp8,fp8,0,0.07946133116881053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,64,128,1,float16,float16,0,0.07524799803892772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,64,0,1,fp8,fp8,0,0.08189866443475087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,64,0,1,float16,float16,0,0.08428266644477844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,64,128,1,float16,fp8,0,0.07366933425267537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,64,128,1,fp8,fp8,0,0.09618133306503296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,64,0,1,float16,fp8,0,0.08413867155710857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,64,128,1,float16,float16,0,0.057578667998313904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,64,0,1,float16,float16,0,0.06277866661548615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,64,128,1,float16,fp8,0,0.05691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,64,0,1,float16,fp8,0,0.06237866481145223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,64,128,1,fp8,fp8,0,0.07686399916807811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,64,0,1,fp8,fp8,0,0.05967999994754791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,64,128,1,float16,float16,0,0.056426664193471275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,64,0,1,float16,float16,0,0.06159466505050659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,64,128,1,float16,fp8,0,0.05728533367315928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,64,128,1,fp8,fp8,0,0.06674666702747345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,64,0,1,float16,fp8,0,0.061850666999816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,64,0,1,fp8,fp8,0,0.058543999989827476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,64,128,1,float16,float16,0,0.05695466697216034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,64,0,1,float16,float16,0,0.06163200239340464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,64,128,1,float16,fp8,0,0.05650666852792104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,64,128,1,fp8,fp8,0,0.06883200009663899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,64,0,1,float16,fp8,0,0.0627040018637975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,64,0,1,fp8,fp8,0,0.05921066800753275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,64,128,1,float16,float16,0,0.05727999905745188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,64,0,1,float16,float16,0,0.061648001273473106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,64,128,1,float16,fp8,0,0.05737066765626272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,64,128,1,fp8,fp8,0,0.06622399886449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,64,128,1,fp8,fp8,0,0.06669866542021434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,64,0,1,float16,fp8,0,0.06188266475995382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,64,0,1,fp8,fp8,0,0.05866666634877523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,64,128,1,float16,float16,0,0.05760533114274343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,64,0,1,float16,float16,0,0.062261333068211876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,64,128,1,float16,fp8,0,0.05754666527112325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,64,0,1,float16,fp8,0,0.06224533418814341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,64,0,1,fp8,fp8,0,0.05863999823729197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,64,128,1,float16,float16,0,1.831125259399414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,64,128,1,float16,fp8,0,1.8282826741536458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,64,0,1,float16,float16,0,2.0788639386494956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,64,0,1,fp8,fp8,0,1.8905760447184246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,64,0,1,float16,fp8,0,2.082853317260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,64,128,1,float16,float16,0,1.866762638092041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,64,128,1,fp8,fp8,0,2.417738596598307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,64,0,1,float16,float16,0,2.112677256266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,64,128,1,float16,fp8,0,1.847343921661377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,64,128,1,fp8,fp8,0,2.429269313812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,64,0,1,fp8,fp8,0,1.923957347869873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,64,0,1,float16,fp8,0,2.0937867164611816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,64,128,1,float16,float16,0,1.9078186353047688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,64,0,1,float16,float16,0,2.1476906140645347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,64,128,1,float16,fp8,0,1.920032024383545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,64,128,1,fp8,fp8,0,2.5268054008483887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,64,128,1,float16,float16,0,1.8719840049743652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,64,0,1,float16,fp8,0,2.1344106992085776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,64,0,1,fp8,fp8,0,2.036677360534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,64,0,1,float16,float16,0,2.132751941680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,64,128,1,float16,fp8,0,1.951013406117757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,64,128,1,float16,float16,0,1.1134133338928223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,64,128,1,float16,fp8,0,1.0743306477864583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,64,0,1,float16,float16,0,1.234282652537028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,64,128,1,fp8,fp8,0,2.557898680369059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,64,0,1,float16,fp8,0,1.2053653399149578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,64,0,1,float16,fp8,0,2.1145973205566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,64,128,1,fp8,fp8,0,1.347808043162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,64,0,1,fp8,fp8,0,2.006890614827474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,64,0,1,fp8,fp8,0,1.0778559843699138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,64,128,1,float16,float16,0,0.9208319981892904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,64,128,1,float16,fp8,0,0.9178933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,64,0,1,float16,float16,0,1.049605369567871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,64,0,1,fp8,fp8,0,0.9600319862365723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,64,0,1,float16,fp8,0,1.0565280119578044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,64,128,1,fp8,fp8,0,1.2274186611175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,64,128,1,float16,float16,0,0.939359982808431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,64,0,1,float16,float16,0,1.0709599653879802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,64,128,1,float16,fp8,0,0.9295093218485514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,64,0,1,float16,fp8,0,1.0496906439463298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,64,0,1,fp8,fp8,0,0.975653330485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,64,128,1,float16,float16,0,0.9670560359954834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,64,128,1,fp8,fp8,0,1.2295093536376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,64,0,1,float16,float16,0,1.094922701517741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,64,128,1,float16,fp8,0,0.9653226534525553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,64,0,1,float16,fp8,0,1.0762080351511638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,64,128,1,fp8,fp8,0,1.2973333199818928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,64,0,1,fp8,fp8,0,1.0049920082092285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,64,128,1,float16,float16,0,0.9610666433970133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,64,0,1,float16,float16,0,1.0905226866404216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,64,128,1,float16,fp8,0,0.9460000197092692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,64,128,1,float16,float16,0,0.5647093454996744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,64,0,1,float16,fp8,0,1.088213364283244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,64,0,1,float16,float16,0,0.6255199909210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,64,128,1,fp8,fp8,0,1.2874240080515544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,64,128,1,float16,fp8,0,0.5533920129140218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,64,0,1,fp8,fp8,0,1.009552001953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,64,128,1,fp8,fp8,0,0.6895306905110677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,64,0,1,float16,fp8,0,0.6106506586074829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,64,0,1,fp8,fp8,0,0.5438826481501261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,64,128,1,float16,float16,0,0.47890134652455646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,64,0,1,float16,float16,0,0.541813333829244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,64,128,1,float16,fp8,0,0.47754132747650146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,64,128,1,fp8,fp8,0,0.6339733203252157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,64,0,1,float16,fp8,0,0.5404320160547892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,64,0,1,fp8,fp8,0,0.4943733215332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,64,128,1,float16,float16,0,0.47782933712005615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,64,0,1,float16,float16,0,0.5493653217951456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,64,128,1,float16,fp8,0,0.4837706486384074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,64,128,1,fp8,fp8,0,0.6418399810791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,64,0,1,float16,fp8,0,0.544981320699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,64,0,1,fp8,fp8,0,0.4917173385620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,64,128,1,float16,float16,0,0.4920213222503662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,64,0,1,float16,float16,0,0.5587093432744344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,64,128,1,float16,fp8,0,0.48520533243815106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,64,128,1,fp8,fp8,0,0.660373330116272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,64,0,1,float16,fp8,0,0.5572160085042318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,64,0,1,fp8,fp8,0,0.5182720025380453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,64,128,1,float16,float16,0,0.49218134085337323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,64,0,1,float16,float16,0,0.5511413415273031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,64,128,1,float16,fp8,0,0.49453866481781006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,64,128,1,float16,float16,0,0.29503466685612995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,64,128,1,fp8,fp8,0,0.6606186628341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,64,0,1,float16,float16,0,0.3288586735725403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,64,0,1,fp8,fp8,0,0.5185386737187704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,64,0,1,float16,fp8,0,0.5509066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,64,0,1,fp8,fp8,0,0.2762986620267232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,64,128,1,float16,fp8,0,0.2887360056241353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,64,128,1,fp8,fp8,0,0.35739731788635254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,64,0,1,float16,fp8,0,0.3216213385264079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,64,128,1,float16,float16,0,0.2515359918276469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,64,0,1,float16,float16,0,0.2834080060323079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,64,128,1,float16,fp8,0,0.2529813249905904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,64,128,1,fp8,fp8,0,0.32944534222284955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,64,0,1,float16,fp8,0,0.2807573278745015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,64,0,1,fp8,fp8,0,0.2521653374036153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,64,128,1,float16,float16,0,0.2513866623242696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,64,0,1,float16,float16,0,0.28402666250864667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,64,128,1,float16,fp8,0,0.25376532475153607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,64,128,1,fp8,fp8,0,0.3381280104319255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,64,0,1,float16,fp8,0,0.28754132986068726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,64,0,1,fp8,fp8,0,0.2528533339500427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,64,128,1,float16,float16,0,0.2569013237953186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,64,0,1,float16,float16,0,0.29264533519744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,64,128,1,float16,fp8,0,0.2563573320706685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,64,128,1,fp8,fp8,0,0.342085321744283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,64,0,1,float16,fp8,0,0.2908800045649211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,64,0,1,fp8,fp8,0,0.2619360089302063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,64,128,1,float16,float16,0,0.2579519947369893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,64,0,1,float16,float16,0,0.2919413248697917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,64,128,1,float16,fp8,0,0.2611040075620015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,64,128,1,fp8,fp8,0,0.34093332290649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,64,0,1,float16,fp8,0,0.29362134138743085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,64,0,1,fp8,fp8,0,0.2625439961751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,64,128,1,fp8,fp8,0,0.19845867156982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,64,128,1,float16,float16,0,0.16058133045832315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,64,0,1,float16,float16,0,0.18279999494552612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,64,128,1,float16,fp8,0,0.15818132956822714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,64,0,1,float16,fp8,0,0.1794453263282776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,64,0,1,fp8,fp8,0,0.14820266763369241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,64,128,1,float16,float16,0,0.1370560030142466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,64,0,1,float16,float16,0,0.15016000469525656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,64,128,1,float16,fp8,0,0.13843733072280884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,64,128,1,fp8,fp8,0,0.1843573252360026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,64,0,1,float16,fp8,0,0.15013866623242697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,64,128,1,float16,fp8,0,0.1393173336982727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,64,0,1,fp8,fp8,0,0.13582932949066162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,64,128,1,float16,float16,0,0.13870933651924133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,64,0,1,float16,float16,0,0.15028267105420431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,64,128,1,fp8,fp8,0,0.18665599822998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,64,0,1,float16,fp8,0,0.14991999665896097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,64,0,1,fp8,fp8,0,0.1389226714769999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,64,128,1,float16,float16,0,0.14285332957903543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,64,0,1,float16,float16,0,0.1532426675160726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,64,128,1,float16,fp8,0,0.14291733503341675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,64,128,1,fp8,fp8,0,0.18786666790644327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,64,0,1,float16,fp8,0,0.15373333295186362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,64,0,1,fp8,fp8,0,0.13858667016029358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,64,128,1,float16,float16,0,0.14500266313552856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,64,0,1,float16,float16,0,0.15500266353289285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,64,128,1,float16,fp8,0,0.1453279952208201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,64,128,1,fp8,fp8,0,0.1893440087636312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,64,0,1,float16,fp8,0,0.15435199936230978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,64,0,1,fp8,fp8,0,0.1404266655445099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,64,128,1,float16,float16,0,0.09206933776537578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,64,0,1,float16,float16,0,0.09559999903043111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,64,128,1,float16,fp8,0,0.09113599856694539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,64,128,1,fp8,fp8,0,0.11877866586049397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,64,0,1,float16,fp8,0,0.09324799974759419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,64,0,1,fp8,fp8,0,0.08562133709589641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,64,128,1,float16,float16,0,0.07766399780909221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,64,0,1,float16,float16,0,0.08240533371766408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,64,128,1,float16,fp8,0,0.0784746656815211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,64,128,1,fp8,fp8,0,0.1039680043856303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,64,0,1,float16,fp8,0,0.08255999783674876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,64,0,1,fp8,fp8,0,0.07622933387756348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,64,128,1,float16,float16,0,0.07747200131416321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,64,0,1,float16,float16,0,0.08358933528264363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,64,128,1,float16,fp8,0,0.08036266764005025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,64,0,1,float16,float16,0,0.08530132969220479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,64,128,1,fp8,fp8,0,0.10849600036938985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,64,0,1,float16,fp8,0,0.08423999945322673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,64,0,1,fp8,fp8,0,0.07713599999745686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,64,128,1,float16,float16,0,0.08159466584523518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,64,128,1,float16,fp8,0,0.08177599807580312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,64,128,1,fp8,fp8,0,0.11046399672826131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,64,0,1,float16,fp8,0,0.08589866757392883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,64,0,1,fp8,fp8,0,0.07850666840871175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,64,128,1,float16,float16,0,0.08438400427500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,64,0,1,float16,float16,0,0.08798399567604065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,64,128,1,float16,fp8,0,0.08297066887219746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,64,128,1,fp8,fp8,0,0.11180800199508667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,64,128,1,fp8,fp8,0,0.07361066838105519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,64,0,1,float16,fp8,0,0.08802666266759236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,64,0,1,fp8,fp8,0,0.08027199904123943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,64,128,1,float16,float16,0,0.05359466870625814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,64,0,1,float16,float16,0,0.054010664423306785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,64,128,1,float16,fp8,0,0.05385600030422211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,64,0,1,float16,fp8,0,0.054378668467203774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,64,0,1,fp8,fp8,0,0.050757333636283875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,64,128,1,float16,float16,0,0.04845866560935974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,64,0,1,float16,float16,0,0.049546668926874794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,64,128,1,float16,fp8,0,0.048565333088239036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,64,128,1,fp8,fp8,0,0.0643039991458257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,64,128,1,fp8,fp8,0,0.06332266827424367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,64,0,1,float16,fp8,0,0.04977599779764811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,64,0,1,fp8,fp8,0,0.04684266448020935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,64,128,1,float16,float16,0,0.05014933149019877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,64,128,1,float16,float16,0,0.04876266419887543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,64,0,1,float16,float16,0,0.050426666935284935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,64,128,1,float16,fp8,0,0.0492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,64,0,1,float16,fp8,0,0.05045866469542185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,64,0,1,fp8,fp8,0,0.047584002216657005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,64,0,1,float16,float16,0,0.05141866703828176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,64,128,1,float16,fp8,0,0.0505920002857844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,64,128,1,fp8,fp8,0,0.06630399823188782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,64,0,1,float16,fp8,0,0.0518506666024526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,64,0,1,fp8,fp8,0,0.04716266691684723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,64,128,1,float16,float16,0,0.05161599814891815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,64,0,1,float16,float16,0,0.052389333645502724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,64,128,1,float16,fp8,0,0.05169066786766052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,64,0,1,float16,float16,0,0.03723733375469843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,64,128,1,fp8,fp8,0,0.06782933572928111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,64,0,1,float16,fp8,0,0.052373334765434265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,64,0,1,fp8,fp8,0,0.04965866605440775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,64,128,1,float16,float16,0,0.03968533376852671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,64,128,1,float16,fp8,0,0.037658666570981346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,64,128,1,fp8,fp8,0,0.04218666752179464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,64,0,1,float16,fp8,0,0.03734400123357773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,64,0,1,fp8,fp8,0,0.03469333300987879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,64,128,1,float16,float16,0,0.03530666728814443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,64,0,1,float16,float16,0,0.03443199892838796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,64,128,1,float16,fp8,0,0.03834133346875509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,64,128,1,fp8,fp8,0,0.04075733323891958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,64,128,1,float16,fp8,0,0.03714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,64,0,1,float16,fp8,0,0.034458667039871216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,64,0,1,fp8,fp8,0,0.032746667663256325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,64,128,1,float16,float16,0,0.03886933376391729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,64,0,1,float16,float16,0,0.03482666611671448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,64,128,1,fp8,fp8,0,0.04091199984153112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,64,0,1,float16,fp8,0,0.03499199946721395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,64,0,1,fp8,fp8,0,0.03252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,64,128,1,float16,float16,0,0.0390079990029335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,64,0,1,float16,float16,0,0.03559466699759165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,64,128,1,float16,fp8,0,0.03996799886226654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,64,128,1,fp8,fp8,0,0.04145599901676178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,64,0,1,float16,fp8,0,0.035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,64,0,1,fp8,fp8,0,0.033439998825391136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,64,128,1,float16,float16,0,0.03947199881076813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,64,0,1,float16,float16,0,0.03653866549332937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,64,128,1,float16,fp8,0,0.03937066594759623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,64,128,1,fp8,fp8,0,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,64,0,1,float16,fp8,0,0.036015999813874565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,64,0,1,fp8,fp8,0,0.0341333324710528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,64,128,1,float16,float16,0,1.7449439366658528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,64,0,1,float16,float16,0,1.7224267323811848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,64,128,1,float16,fp8,0,1.7384586334228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,64,0,1,fp8,fp8,0,1.5320852597554524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,64,0,1,float16,fp8,0,1.7268586158752441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,64,128,1,float16,float16,0,1.7724960645039876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,64,128,1,fp8,fp8,0,2.2742932637532554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,64,128,1,float16,fp8,0,1.7628533045450847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,64,0,1,float16,float16,0,1.7387199401855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,64,0,1,float16,fp8,0,1.749226729075114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,64,0,1,fp8,fp8,0,1.569551944732666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,64,128,1,float16,float16,0,1.8044320742289226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,64,128,1,fp8,fp8,0,2.306037267049154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,64,0,1,float16,float16,0,1.7821067174275715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,64,128,1,float16,fp8,0,1.8271840413411458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,64,0,1,float16,fp8,0,1.8046293258666992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,64,0,1,fp8,fp8,0,1.637056032816569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,64,128,1,fp8,fp8,0,2.3917919794718423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,64,128,1,float16,float16,0,1.8212693532307942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,64,0,1,float16,float16,0,1.783664067586263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,64,128,1,float16,fp8,0,1.7710986137390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,64,128,1,fp8,fp8,0,2.3993333180745444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,64,128,1,float16,float16,0,1.0623573462168376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,64,0,1,float16,fp8,0,1.8321812947591145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,64,0,1,float16,float16,0,1.0589973131815593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,64,128,1,float16,fp8,0,1.0348426500956218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,64,0,1,fp8,fp8,0,1.6823147137959797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,64,0,1,float16,fp8,0,1.0085333188374836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,64,128,1,fp8,fp8,0,1.2793653011322021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,64,0,1,fp8,fp8,0,0.8919680118560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,64,128,1,float16,float16,0,0.8880320390065511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,64,0,1,float16,float16,0,0.8731573422749838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,64,128,1,float16,fp8,0,0.8846826553344727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,64,0,1,float16,fp8,0,0.8696106274922689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,64,0,1,fp8,fp8,0,0.7631146907806396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,64,128,1,fp8,fp8,0,1.1436426639556885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,64,128,1,float16,float16,0,0.8859146436055502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,64,0,1,float16,float16,0,0.8774080276489258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,64,128,1,float16,fp8,0,0.8889386653900146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,64,0,1,float16,fp8,0,0.8692906697591146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,64,128,1,fp8,fp8,0,1.1837866306304932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,64,0,1,fp8,fp8,0,0.7896533012390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,64,128,1,float16,float16,0,0.9060640335083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,64,0,1,float16,float16,0,0.884389321009318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,64,128,1,float16,fp8,0,0.9107360045115153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,64,0,1,float16,fp8,0,0.8928639888763428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,64,128,1,fp8,fp8,0,1.2157066663106282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,64,0,1,fp8,fp8,0,0.8259039719899496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,64,128,1,float16,float16,0,0.9039093653361002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,64,0,1,float16,float16,0,0.9035893281300863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,64,128,1,float16,fp8,0,0.9043893019358317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,64,0,1,float16,fp8,0,0.9066720008850098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,64,128,1,float16,float16,0,0.5390133460362753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,64,128,1,fp8,fp8,0,1.2043999830881755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,64,0,1,float16,float16,0,0.5304319858551025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,64,0,1,fp8,fp8,0,0.8353866736094157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,64,128,1,float16,fp8,0,0.5257173379262289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,64,0,1,float16,fp8,0,0.5168586572011312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,64,128,1,fp8,fp8,0,0.6465226809183756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,64,0,1,fp8,fp8,0,0.44979198773701984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,64,128,1,float16,float16,0,0.4561760028203328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,64,0,1,float16,float16,0,0.44045865535736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,64,128,1,float16,fp8,0,0.45637865861256915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,64,128,1,fp8,fp8,0,0.5988106727600098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,64,0,1,float16,fp8,0,0.44147201379140216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,64,128,1,float16,float16,0,0.46083199977874756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,64,0,1,fp8,fp8,0,0.39420799414316815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,64,0,1,float16,float16,0,0.4450613260269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,64,128,1,float16,fp8,0,0.4615519841512044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,64,128,1,fp8,fp8,0,0.6045600175857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,64,0,1,float16,fp8,0,0.45114131768544513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,64,0,1,fp8,fp8,0,0.40134934584299725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,64,128,1,float16,float16,0,0.46855465571085614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,64,0,1,float16,float16,0,0.4556693236033122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,64,128,1,float16,fp8,0,0.46407465140024823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,64,0,1,float16,fp8,0,0.4612586498260498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,64,128,1,fp8,fp8,0,0.6223413149515787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,64,128,1,float16,float16,0,0.46612799167633057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,64,0,1,fp8,fp8,0,0.421776016553243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,64,0,1,float16,float16,0,0.4625226656595866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,64,128,1,float16,fp8,0,0.4673866828282674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,64,128,1,float16,float16,0,0.2820213238398234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,64,128,1,fp8,fp8,0,0.6167093515396118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,64,0,1,float16,fp8,0,0.463370680809021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,64,0,1,fp8,fp8,0,0.42401599884033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,64,0,1,float16,float16,0,0.27772800127665204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,64,128,1,float16,fp8,0,0.27829867601394653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,64,128,1,fp8,fp8,0,0.3388426701227824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,64,0,1,float16,fp8,0,0.26793066660563153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,64,0,1,fp8,fp8,0,0.23349332809448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,64,128,1,float16,float16,0,0.24011200666427612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,64,0,1,float16,float16,0,0.2345973253250122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,64,128,1,float16,fp8,0,0.24051199356714884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,64,128,1,fp8,fp8,0,0.31304534276326496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,64,0,1,float16,fp8,0,0.23119467496871948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,64,0,1,fp8,fp8,0,0.2030079960823059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,64,128,1,float16,float16,0,0.24085867404937744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,64,0,1,float16,float16,0,0.23763734102249146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,64,128,1,float16,fp8,0,0.23931733767191568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,64,128,1,fp8,fp8,0,0.3185759981473287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,64,0,1,float16,fp8,0,0.23426133394241333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,64,0,1,fp8,fp8,0,0.20665599902470908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,64,128,1,float16,float16,0,0.2448213299115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,64,0,1,float16,float16,0,0.24170132478078207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,64,128,1,float16,fp8,0,0.24766399463017783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,64,128,1,fp8,fp8,0,0.3190133372942607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,64,0,1,float16,fp8,0,0.2408693234125773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,64,0,1,fp8,fp8,0,0.21156267325083414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,64,128,1,float16,float16,0,0.2453813354174296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,64,0,1,float16,float16,0,0.2395253380139669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,64,128,1,float16,fp8,0,0.24888533353805542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,64,128,1,fp8,fp8,0,0.32094399134318036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,64,0,1,float16,float16,0,0.16319466630617777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,64,128,1,float16,fp8,0,0.15147200226783752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,64,0,1,float16,fp8,0,0.2411200006802877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,64,128,1,float16,float16,0,0.1544373333454132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,64,0,1,fp8,fp8,0,0.2142933408419291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,64,128,1,fp8,fp8,0,0.18901334206263223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,64,0,1,float16,fp8,0,0.15676800409952799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,64,0,1,fp8,fp8,0,0.12601600090662637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,64,128,1,float16,float16,0,0.13114666938781738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,64,0,1,float16,float16,0,0.12541332840919495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,64,128,1,float16,fp8,0,0.13199466466903687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,64,128,1,fp8,fp8,0,0.17465599377950033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,64,0,1,float16,fp8,0,0.12658133109410605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,64,0,1,fp8,fp8,0,0.11369599898656209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,64,128,1,float16,float16,0,0.1332586705684662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,64,0,1,float16,float16,0,0.12732799847920737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,64,128,1,float16,fp8,0,0.13274133205413818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,64,128,1,fp8,fp8,0,0.17382933696111044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,64,0,1,float16,fp8,0,0.1276693344116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,64,0,1,fp8,fp8,0,0.11617066462834676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,64,128,1,float16,float16,0,0.13768000404040018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,64,0,1,float16,float16,0,0.13032000263532004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,64,128,1,float16,fp8,0,0.1374666690826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,64,128,1,fp8,fp8,0,0.17813332875569662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,64,0,1,float16,fp8,0,0.12997866670290628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,64,0,1,fp8,fp8,0,0.11762666702270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,64,128,1,float16,float16,0,0.13961600263913473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,64,128,1,float16,fp8,0,0.1383573313554128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,64,0,1,float16,float16,0,0.13321600357691446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,64,128,1,fp8,fp8,0,0.18117332458496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,64,0,1,float16,fp8,0,0.13126400113105774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,64,0,1,fp8,fp8,0,0.1179039975007375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,64,128,1,float16,float16,0,0.08945066730181377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,64,128,1,float16,float16,0,0.07612800101439159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,64,0,1,float16,float16,0,0.08318399886290233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,64,128,1,float16,fp8,0,0.08853866656621297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,64,128,1,fp8,fp8,0,0.11331733067830403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,64,0,1,float16,fp8,0,0.07221866647402446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,64,0,1,float16,fp8,0,0.08205333352088928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,64,0,1,fp8,fp8,0,0.07450133562088013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,64,0,1,float16,float16,0,0.07117333511511485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,64,128,1,float16,fp8,0,0.07621333499749501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,64,128,1,fp8,fp8,0,0.09698667128880818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,64,0,1,fp8,fp8,0,0.06450133522351582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,64,128,1,float16,float16,0,0.07576533158620198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,64,0,1,float16,float16,0,0.07181333502133687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,64,128,1,float16,fp8,0,0.07690666615962982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,64,128,1,fp8,fp8,0,0.10075733065605164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,64,0,1,float16,fp8,0,0.07197333375612895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,64,0,1,fp8,fp8,0,0.06522666911284129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,64,128,1,float16,float16,0,0.0782239983479182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,64,0,1,float16,float16,0,0.07372800012429555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,64,128,1,float16,fp8,0,0.0782773345708847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,64,128,1,fp8,fp8,0,0.10340799887975057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,64,128,1,float16,fp8,0,0.08166400094827016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,64,0,1,float16,fp8,0,0.0737066666285197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,64,0,1,fp8,fp8,0,0.06849599877993266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,64,128,1,float16,float16,0,0.0806933343410492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,64,128,1,float16,float16,0,0.052426666021347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,64,0,1,float16,float16,0,0.07527466615041097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,64,128,1,fp8,fp8,0,0.10579733053843181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,64,0,1,float16,fp8,0,0.07620266576608022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,64,0,1,fp8,fp8,0,0.0695253312587738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,64,0,1,float16,float16,0,0.04846400022506714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,64,128,1,float16,fp8,0,0.05150933563709259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,64,128,1,fp8,fp8,0,0.06993066767851512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,64,0,1,float16,fp8,0,0.047269334395726524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,64,0,1,fp8,fp8,0,0.044351999958356224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,64,128,1,float16,float16,0,0.047370667258898415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,64,0,1,float16,float16,0,0.04460800190766653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,64,128,1,float16,fp8,0,0.04790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,64,128,1,fp8,fp8,0,0.061050668358802795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,64,0,1,float16,fp8,0,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,64,0,1,fp8,fp8,0,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,64,0,1,float16,fp8,0,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,64,128,1,float16,float16,0,0.048026666045188904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,64,0,1,float16,float16,0,0.04389866689840952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,64,128,1,float16,fp8,0,0.04829333225886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,64,128,1,fp8,fp8,0,0.06280000011126201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,64,0,1,fp8,fp8,0,0.040181333820025124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,64,128,1,float16,float16,0,0.04948799808820089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,64,0,1,float16,float16,0,0.04489066700140635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,64,128,1,float16,fp8,0,0.049141332507133484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,64,128,1,fp8,fp8,0,0.06252799928188324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,64,0,1,float16,fp8,0,0.0462666650613149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,64,0,1,fp8,fp8,0,0.040864000717798867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,64,128,1,float16,float16,0,0.04969066878159841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,64,0,1,float16,float16,0,0.0458186666170756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,64,128,1,float16,fp8,0,0.05035733183224996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,64,128,1,fp8,fp8,0,0.06298666695753734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,64,0,1,float16,fp8,0,0.04637866715590159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,64,0,1,fp8,fp8,0,0.042768001556396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,64,128,1,float16,float16,0,0.03982933362325033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,64,0,1,fp8,fp8,0,0.03150933235883713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,64,0,1,float16,float16,0,0.033439998825391136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,64,128,1,float16,fp8,0,0.03994666785001755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,64,128,1,fp8,fp8,0,0.04233066737651825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,64,0,1,float16,fp8,0,0.03426666557788849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,64,128,1,float16,float16,0,0.03469866762558619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,64,0,1,float16,float16,0,0.03193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,64,128,1,float16,float16,0,0.03477333237727483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,64,128,1,float16,fp8,0,0.035029334326585136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,64,128,1,fp8,fp8,0,0.04008000095685323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,64,0,1,float16,fp8,0,0.032245332996050514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,64,0,1,fp8,fp8,0,0.030042665700117748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,64,0,1,float16,float16,0,0.032186667124430336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,64,128,1,float16,fp8,0,0.03534399966398875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,64,128,1,fp8,fp8,0,0.04127466678619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,64,0,1,float16,fp8,0,0.032858667274316154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,64,0,1,fp8,fp8,0,0.029909332593282063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,64,128,1,float16,float16,0,0.038431999584039055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,64,0,1,float16,float16,0,0.03306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,64,128,1,float16,fp8,0,0.03572266548871994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,64,128,1,fp8,fp8,0,0.04230933388074239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,64,0,1,float16,fp8,0,0.033029332756996155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,64,0,1,fp8,fp8,0,0.03136000037193298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,64,128,1,float16,float16,0,0.0386559988061587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,64,0,1,float16,float16,0,0.033999999364217125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,64,128,1,float16,fp8,0,0.03868266691764196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,64,128,1,fp8,fp8,0,0.0421973317861557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,64,0,1,float16,fp8,0,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,64,0,1,float16,fp8,0,0.024405332903067272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,64,0,1,fp8,fp8,0,0.03172266731659571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,64,128,1,float16,float16,0,0.02641066660483678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,64,0,1,float16,float16,0,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,64,128,1,float16,fp8,0,0.026522666215896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,64,128,1,fp8,fp8,0,0.03345600018898646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,64,0,1,fp8,fp8,0,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,64,128,1,float16,float16,0,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,64,0,1,float16,float16,0,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,64,0,1,float16,float16,0,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,64,128,1,float16,fp8,0,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,64,128,1,fp8,fp8,0,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,64,0,1,float16,fp8,0,0.022863999009132385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,64,0,1,fp8,fp8,0,0.021888000269730885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,64,128,1,float16,float16,0,0.025472000241279602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,64,128,1,float16,fp8,0,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,64,128,1,fp8,fp8,0,0.031925333042939506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,64,0,1,float16,fp8,0,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,64,0,1,fp8,fp8,0,0.022255999346574146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,64,128,1,float16,float16,0,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,64,0,1,float16,float16,0,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,64,128,1,float16,fp8,0,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,64,128,1,fp8,fp8,0,0.03230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,64,0,1,float16,fp8,0,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,64,0,1,fp8,fp8,0,0.022367998957633972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,64,128,1,float16,float16,0,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,64,0,1,float16,float16,0,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,64,128,1,float16,fp8,0,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,64,128,1,fp8,fp8,0,0.03230399886767069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,64,0,1,float16,fp8,0,0.0236160010099411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,64,0,1,fp8,fp8,0,0.022848000129063923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,64,128,1,float16,float16,0,0.7353920141855875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,64,0,1,float16,float16,0,0.7200906276702881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,64,128,1,float16,fp8,0,0.7323466936747233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,64,0,1,fp8,fp8,0,0.6605439980824789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,64,0,1,float16,fp8,0,0.7136639753977457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,64,128,1,fp8,fp8,0,0.9964640140533447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,64,128,1,float16,float16,0,0.7516960302988688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,64,0,1,float16,float16,0,0.7266133626302084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,64,128,1,float16,fp8,0,0.7483519713083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,64,0,1,float16,fp8,0,0.7249013582865397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,64,128,1,fp8,fp8,0,1.0173760255177815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,64,0,1,fp8,fp8,0,0.6745226383209229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,64,128,1,float16,float16,0,0.7787520090738932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,64,0,1,float16,float16,0,0.7690026760101318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,64,128,1,float16,fp8,0,0.7736533482869467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,64,128,1,fp8,fp8,0,1.05076265335083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,64,0,1,float16,fp8,0,0.7567893664042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,64,0,1,fp8,fp8,0,0.7171520392100016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,64,128,1,float16,float16,0,0.7767999966939291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,64,0,1,float16,float16,0,0.7556853294372559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,64,128,1,float16,fp8,0,0.773039976755778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,64,128,1,float16,float16,0,0.4713919957478841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,64,0,1,float16,fp8,0,0.756879965464274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,64,128,1,fp8,fp8,0,1.058282693227132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,64,0,1,float16,float16,0,0.4625600179036458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,64,0,1,fp8,fp8,0,0.7193866570790609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,64,128,1,float16,fp8,0,0.4588586489359538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,64,128,1,fp8,fp8,0,0.5498133500417074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,64,0,1,float16,fp8,0,0.44777599970499676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,64,0,1,fp8,fp8,0,0.38151999314626056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,64,128,1,float16,float16,0,0.3837493260701497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,64,0,1,float16,float16,0,0.3699306646982829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,64,128,1,float16,fp8,0,0.3776959975560506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,64,0,1,float16,fp8,0,0.36870400110880536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,64,128,1,fp8,fp8,0,0.5078346729278564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,64,0,1,fp8,fp8,0,0.3373066584269206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,64,128,1,float16,float16,0,0.38389333089192706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,64,0,1,float16,float16,0,0.3806346654891968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,64,128,1,float16,fp8,0,0.3837333520253499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,64,128,1,fp8,fp8,0,0.508346676826477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,64,0,1,float16,fp8,0,0.3738986651102702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,64,0,1,fp8,fp8,0,0.3463840087254842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,64,128,1,float16,float16,0,0.3977866570154826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,64,0,1,float16,float16,0,0.38678399721781415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,64,128,1,float16,fp8,0,0.39660799503326416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,64,0,1,float16,fp8,0,0.3852693239847819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,64,128,1,fp8,fp8,0,0.5298773447672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,64,0,1,fp8,fp8,0,0.36286401748657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,64,128,1,float16,float16,0,0.40159467856089276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,64,0,1,float16,float16,0,0.39766931533813477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,64,128,1,float16,fp8,0,0.4006613492965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,64,0,1,float16,fp8,0,0.39373334248860675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,64,128,1,float16,float16,0,0.2502666711807251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,64,0,1,fp8,fp8,0,0.3631306489308675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,64,128,1,fp8,fp8,0,0.5317759911219279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,64,0,1,float16,float16,0,0.2467093269030253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,64,128,1,float16,fp8,0,0.24413333336512247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,64,128,1,fp8,fp8,0,0.2861066659291585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,64,0,1,float16,fp8,0,0.24042133490244547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,64,0,1,fp8,fp8,0,0.2030400037765503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,64,128,1,float16,float16,0,0.1978879968325297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,64,0,1,fp8,fp8,0,0.1811093290646871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,64,0,1,float16,float16,0,0.19285333156585693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,64,128,1,float16,fp8,0,0.1982826590538025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,64,128,1,fp8,fp8,0,0.2619679967562358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,64,0,1,float16,fp8,0,0.19308799505233765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,64,128,1,float16,float16,0,0.20145599047342935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,64,0,1,float16,float16,0,0.19774399201075235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,64,128,1,float16,fp8,0,0.2010400096575419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,64,128,1,fp8,fp8,0,0.26946133375167847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,64,0,1,float16,fp8,0,0.19596266746520996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,64,0,1,fp8,fp8,0,0.18231467405954996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,64,128,1,float16,float16,0,0.2099413275718689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,64,0,1,float16,float16,0,0.20523732900619507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,64,128,1,float16,fp8,0,0.20784533023834229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,64,128,1,fp8,fp8,0,0.2716853419939677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,64,0,1,float16,fp8,0,0.20396800835927328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,64,0,1,fp8,fp8,0,0.18720000982284546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,64,128,1,float16,float16,0,0.21228800217310587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,64,0,1,float16,float16,0,0.20894400278727213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,64,128,1,float16,fp8,0,0.2135146657625834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,64,128,1,fp8,fp8,0,0.27268799146016437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,64,0,1,float16,fp8,0,0.20930665731430054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,64,128,1,fp8,fp8,0,0.15229866902033487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,64,0,1,fp8,fp8,0,0.1920479933420817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,64,128,1,float16,float16,0,0.14010133345921835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,64,0,1,float16,float16,0,0.13769599795341492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,64,128,1,float16,fp8,0,0.1360586682955424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,64,0,1,float16,fp8,0,0.13497599959373474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,64,0,1,fp8,fp8,0,0.11254933476448059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,64,128,1,float16,float16,0,0.10754666725794475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,64,0,1,float16,float16,0,0.10425600409507751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,64,128,1,float16,fp8,0,0.10776533683141072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,64,128,1,fp8,fp8,0,0.1426400045553843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,64,0,1,float16,fp8,0,0.10429867108662923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,64,0,1,fp8,fp8,0,0.10099732875823975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,64,128,1,float16,float16,0,0.10918399691581726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,64,0,1,float16,float16,0,0.10691199700037639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,64,128,1,float16,fp8,0,0.10924800237019856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,64,128,1,fp8,fp8,0,0.14379200339317322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,64,0,1,float16,fp8,0,0.10707199573516846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,64,0,1,fp8,fp8,0,0.10384000341097514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,64,128,1,float16,float16,0,0.11446932951609294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,64,0,1,float16,float16,0,0.1125333309173584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,64,128,1,float16,fp8,0,0.11361066500345866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,64,128,1,fp8,fp8,0,0.14627200365066528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,64,0,1,float16,fp8,0,0.11199999849001567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,64,128,1,float16,fp8,0,0.11826133728027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,64,0,1,fp8,fp8,0,0.10655466715494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,64,128,1,float16,float16,0,0.11910933256149292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,64,128,1,float16,float16,0,0.08339200417200725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,64,0,1,fp8,fp8,0,0.10568533341089885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,64,0,1,float16,float16,0,0.11646933356920879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,64,128,1,fp8,fp8,0,0.08668266733487447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,64,128,1,fp8,fp8,0,0.1465226709842682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,64,0,1,float16,fp8,0,0.11687466502189636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,64,0,1,float16,float16,0,0.08249600231647491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,64,0,1,float16,float16,0,0.06303466856479645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,64,128,1,float16,fp8,0,0.08117333551247914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,64,0,1,float16,fp8,0,0.07949333389600118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,64,0,1,fp8,fp8,0,0.06738666693369548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,64,128,1,float16,float16,0,0.06259199976921082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,64,128,1,float16,float16,0,0.06435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,64,128,1,float16,fp8,0,0.06347733239332835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,64,128,1,fp8,fp8,0,0.07765333354473114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,64,0,1,float16,fp8,0,0.06316799918810527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,64,0,1,fp8,fp8,0,0.057333335280418396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,64,0,1,float16,float16,0,0.0652159998814265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,64,128,1,float16,fp8,0,0.06390400230884552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,64,128,1,fp8,fp8,0,0.07920533418655396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,64,0,1,float16,fp8,0,0.0633546660343806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,64,128,1,fp8,fp8,0,0.08134399851163228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,64,0,1,fp8,fp8,0,0.05895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,64,128,1,float16,float16,0,0.06600533425807953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,64,0,1,float16,float16,0,0.06592000027497609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,64,128,1,float16,fp8,0,0.0654720018307368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,64,0,1,float16,fp8,0,0.06573866804440816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,64,128,1,float16,fp8,0,0.06605866551399231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,64,0,1,fp8,fp8,0,0.05987200140953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,64,128,1,float16,float16,0,0.06694399813810985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,64,0,1,float16,float16,0,0.06864533325036366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,64,128,1,fp8,fp8,0,0.08255999783674876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,64,0,1,float16,fp8,0,0.06834666430950165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,64,128,1,fp8,fp8,0,0.050517335534095764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,64,0,1,fp8,fp8,0,0.06122666597366333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,64,128,1,float16,float16,0,0.04173333446184794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,64,0,1,float16,float16,0,0.04101333270470301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,64,128,1,float16,fp8,0,0.04238933324813843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,64,0,1,float16,fp8,0,0.04142933338880539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,64,128,1,fp8,fp8,0,0.04598933458328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,64,0,1,fp8,fp8,0,0.039647998909155525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,64,128,1,float16,float16,0,0.038704000413417816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,64,0,1,float16,float16,0,0.036901332437992096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,64,128,1,float16,fp8,0,0.038405333956082664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,64,0,1,float16,fp8,0,0.03693866729736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,64,128,1,float16,float16,0,0.037946666280428566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,64,0,1,fp8,fp8,0,0.03554133325815201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,64,0,1,float16,float16,0,0.03721600025892258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,64,128,1,float16,fp8,0,0.03920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,64,128,1,fp8,fp8,0,0.04650133351484934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,64,128,1,float16,fp8,0,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,64,0,1,float16,fp8,0,0.03762666632731756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,64,0,1,fp8,fp8,0,0.036330667634805046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,64,128,1,float16,float16,0,0.039162665605545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,64,0,1,float16,float16,0,0.03835733234882355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,64,128,1,fp8,fp8,0,0.04783466458320618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,64,0,1,float16,fp8,0,0.03905066599448522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,64,0,1,fp8,fp8,0,0.03771200031042099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,64,0,1,fp8,fp8,0,0.03678400069475174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,64,128,1,float16,float16,0,0.0403413325548172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,64,0,1,float16,float16,0,0.03957866628964742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,64,128,1,float16,fp8,0,0.040175999204317726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,64,128,1,fp8,fp8,0,0.04692799846331278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,64,0,1,float16,fp8,0,0.03932799895604452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,64,128,1,float16,float16,0,0.031258667508761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,64,0,1,float16,float16,0,0.031199999153614044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,64,128,1,float16,fp8,0,0.031311998764673867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,64,128,1,fp8,fp8,0,0.035749333600203194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,64,0,1,float16,fp8,0,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,64,0,1,fp8,fp8,0,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,64,128,1,float16,float16,0,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,64,0,1,float16,float16,0,0.028207999964555103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,64,128,1,float16,fp8,0,0.028602667152881622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,64,128,1,fp8,fp8,0,0.035088000198205314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,64,0,1,float16,fp8,0,0.028442665934562683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,64,0,1,fp8,fp8,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,64,128,1,float16,float16,0,0.028543998797734577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,64,0,1,float16,float16,0,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,64,128,1,float16,fp8,0,0.028981332977612812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,64,128,1,fp8,fp8,0,0.035114665826161705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,64,0,1,float16,fp8,0,0.02845333268245061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,64,0,1,fp8,fp8,0,0.028245332340399425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,64,128,1,float16,float16,0,0.029109333952267964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,64,0,1,float16,float16,0,0.029232000311215717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,64,128,1,float16,fp8,0,0.02932800104220708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,64,128,1,fp8,fp8,0,0.0351946676770846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,64,0,1,float16,fp8,0,0.02906133234500885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,64,0,1,fp8,fp8,0,0.028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,64,128,1,float16,float16,0,0.029930666089057922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,64,0,1,float16,float16,0,0.029824001093705494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,64,128,1,float16,fp8,0,0.02985599885384242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,64,128,1,fp8,fp8,0,0.035877334574858345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,64,0,1,float16,fp8,0,0.02916266769170761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,64,0,1,fp8,fp8,0,0.02884799987077713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,64,128,1,float16,float16,0,0.02027200038234393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,64,0,1,float16,float16,0,0.019952000429232914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,64,128,1,float16,fp8,0,0.020453333854675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,64,128,1,fp8,fp8,0,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,64,0,1,float16,fp8,0,0.020288000504175823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,64,0,1,fp8,fp8,0,0.020597333709398907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,64,128,1,float16,float16,0,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,64,0,1,float16,float16,0,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,64,128,1,float16,fp8,0,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,64,128,1,fp8,fp8,0,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,64,0,1,float16,fp8,0,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,64,0,1,fp8,fp8,0,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,64,128,1,float16,float16,0,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,64,0,1,float16,float16,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,64,128,1,float16,fp8,0,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,64,128,1,fp8,fp8,0,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,64,0,1,float16,fp8,0,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,64,0,1,fp8,fp8,0,0.019797333826621372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,64,128,1,float16,float16,0,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,64,0,1,float16,float16,0,0.01859733338157336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,64,128,1,float16,fp8,0,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,64,0,1,float16,float16,0,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,64,128,1,fp8,fp8,0,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,64,0,1,float16,fp8,0,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,64,0,1,fp8,fp8,0,0.02038399999340375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,64,128,1,float16,float16,0,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,64,0,1,fp8,fp8,0,0.020293333878119785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,64,128,1,float16,fp8,0,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,64,128,1,fp8,fp8,0,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,64,0,1,float16,fp8,0,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,64,128,1,float16,float16,0,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,64,0,1,float16,float16,0,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,64,128,1,float16,fp8,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,64,128,1,fp8,fp8,0,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,64,0,1,float16,fp8,0,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,64,0,1,fp8,fp8,0,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,64,128,1,float16,float16,0,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,64,0,1,float16,float16,0,0.01618133361140887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,64,128,1,float16,fp8,0,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,64,128,1,fp8,fp8,0,0.02277333289384842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,64,0,1,float16,fp8,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,64,0,1,fp8,fp8,0,0.01829333355029424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,64,128,1,float16,float16,0,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,64,0,1,float16,float16,0,0.016437333077192307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,64,128,1,float16,fp8,0,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,64,128,1,fp8,fp8,0,0.022815999885400135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,64,0,1,float16,fp8,0,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,64,0,1,fp8,fp8,0,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,64,128,1,float16,float16,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,64,0,1,float16,float16,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,64,128,1,float16,fp8,0,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,64,128,1,fp8,fp8,0,0.022277332842350006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,64,0,1,float16,fp8,0,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,64,0,1,fp8,fp8,0,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,64,128,1,float16,float16,0,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,64,0,1,float16,float16,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,64,128,1,float16,fp8,0,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,64,128,1,fp8,fp8,0,0.023269332945346832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,64,0,1,float16,fp8,0,0.01674666628241539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,64,0,1,fp8,fp8,0,0.020186666399240494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,64,128,1,float16,float16,0,0.2770079970359802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,64,0,1,float16,float16,0,0.2786719997723897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,64,128,1,fp8,fp8,0,0.3293066620826721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,64,128,1,float16,fp8,0,0.27457066377003986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,64,0,1,float16,fp8,0,0.2739466627438863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,64,0,1,fp8,fp8,0,0.32979732751846313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,64,128,1,float16,float16,0,0.2850186626116435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,64,0,1,float16,float16,0,0.2842880090077718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,64,128,1,float16,fp8,0,0.2749386628468831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,64,0,1,float16,fp8,0,0.27636800209681195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,64,128,1,fp8,fp8,0,0.3330186605453491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,64,0,1,fp8,fp8,0,0.3381813367207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,64,128,1,float16,float16,0,0.2924319903055827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,64,0,1,float16,float16,0,0.292522668838501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,64,128,1,float16,fp8,0,0.30131733417510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,64,128,1,fp8,fp8,0,0.3661653200785319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,64,0,1,float16,fp8,0,0.3004746635754903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,64,0,1,fp8,fp8,0,0.3649386564890544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,64,128,1,float16,float16,0,0.2973066568374634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,64,0,1,float16,float16,0,0.2978293299674988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,64,128,1,float16,fp8,0,0.2943093379338582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,64,128,1,fp8,fp8,0,0.3641759951909383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,64,0,1,float16,fp8,0,0.29657600323359173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,64,128,1,float16,float16,0,0.20479466517766318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,64,0,1,fp8,fp8,0,0.3659573396046956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,64,0,1,float16,float16,0,0.20433600743611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,64,128,1,float16,fp8,0,0.19795199235280356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,64,128,1,fp8,fp8,0,0.20138667027155557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,64,0,1,float16,fp8,0,0.19741866985956827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,64,0,1,fp8,fp8,0,0.20321067174275717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,64,128,1,float16,float16,0,0.14645333091417947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,64,0,1,float16,float16,0,0.14619732896486917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,64,128,1,float16,fp8,0,0.14507200320561728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,64,0,1,float16,float16,0,0.14802666505177817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,64,128,1,fp8,fp8,0,0.17787732680638632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,64,0,1,float16,fp8,0,0.14573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,64,0,1,fp8,fp8,0,0.17624000708262125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,64,128,1,float16,float16,0,0.14934933185577393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,64,128,1,float16,fp8,0,0.14718400438626608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,64,0,1,float16,fp8,0,0.1460533340771993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,64,128,1,fp8,fp8,0,0.1770346760749817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,64,0,1,fp8,fp8,0,0.17575466632843018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,64,128,1,float16,float16,0,0.15428266922632852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,64,0,1,float16,float16,0,0.1532266636689504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,64,128,1,float16,fp8,0,0.15381866693496704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,64,128,1,fp8,fp8,0,0.19291732708613077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,64,0,1,float16,fp8,0,0.15345600247383118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,64,0,1,fp8,fp8,0,0.19172799587249756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,64,128,1,float16,float16,0,0.15853333473205566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,64,0,1,float16,float16,0,0.15897599856058756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,64,128,1,float16,fp8,0,0.15601066748301187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,64,128,1,fp8,fp8,0,0.18890132506688437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,64,0,1,float16,fp8,0,0.15787200133005777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,64,128,1,float16,float16,0,0.11327999830245972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,64,0,1,fp8,fp8,0,0.19040000438690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,64,0,1,float16,float16,0,0.11231999595959981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,64,128,1,float16,fp8,0,0.10962133606274922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,64,128,1,float16,float16,0,0.08212266862392426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,64,128,1,fp8,fp8,0,0.11097066601117452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,64,0,1,float16,fp8,0,0.11012267072995503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,64,128,1,fp8,fp8,0,0.09742400050163269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,64,0,1,fp8,fp8,0,0.11249599854151408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,64,0,1,float16,float16,0,0.08210666477680206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,64,128,1,float16,fp8,0,0.08197866876920064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,64,0,1,float16,fp8,0,0.08160000046094258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,64,0,1,fp8,fp8,0,0.09852266311645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,64,128,1,float16,float16,0,0.0839466651280721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,64,0,1,float16,float16,0,0.08369599779446919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,64,128,1,float16,fp8,0,0.0832479993502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,64,128,1,fp8,fp8,0,0.09936533371607463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,64,0,1,float16,fp8,0,0.08262399832407634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,64,0,1,fp8,fp8,0,0.09903466701507568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,64,128,1,float16,float16,0,0.08802133798599243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,64,0,1,float16,float16,0,0.08747733632723491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,64,128,1,float16,fp8,0,0.08710933725039165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,64,128,1,fp8,fp8,0,0.10724266370137532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,64,0,1,float16,fp8,0,0.08754133184750874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,64,0,1,fp8,fp8,0,0.10614400108655293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,64,128,1,float16,float16,0,0.09129599730173747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,64,0,1,float16,float16,0,0.09134933352470398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,64,128,1,float16,fp8,0,0.09026133020718892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,64,128,1,fp8,fp8,0,0.1048426628112793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,64,0,1,float16,fp8,0,0.09112000465393066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,64,0,1,fp8,fp8,0,0.104912002881368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,64,128,1,float16,float16,0,0.06381333371003468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,64,0,1,float16,float16,0,0.06382933259010315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,64,128,1,float16,fp8,0,0.06159466505050659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,64,128,1,fp8,fp8,0,0.06490666667620341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,64,0,1,float16,fp8,0,0.06164266665776571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,64,0,1,fp8,fp8,0,0.06613866488138835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,64,128,1,float16,float16,0,0.047024001677831016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,64,0,1,float16,float16,0,0.046485334634780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,64,128,1,float16,fp8,0,0.046341334780057274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,64,128,1,fp8,fp8,0,0.05624000231424967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,64,0,1,float16,fp8,0,0.046816001335779824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,64,0,1,fp8,fp8,0,0.05611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,64,128,1,float16,float16,0,0.04816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,64,0,1,fp8,fp8,0,0.05819733440876007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,64,0,1,float16,float16,0,0.047370667258898415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,64,128,1,float16,fp8,0,0.0468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,64,128,1,fp8,fp8,0,0.05770133435726166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,64,0,1,float16,fp8,0,0.04711466530958811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,64,128,1,float16,float16,0,0.049413333336512245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,64,0,1,float16,float16,0,0.04957333207130432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,64,128,1,float16,fp8,0,0.04949333270390829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,64,128,1,fp8,fp8,0,0.06112533311049143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,64,0,1,float16,fp8,0,0.04797333478927612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,64,0,1,fp8,fp8,0,0.06121066709359487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,64,128,1,float16,float16,0,0.05065066615740458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,64,0,1,float16,float16,0,0.050981332858403526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,64,128,1,float16,fp8,0,0.050623998045921326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,64,128,1,fp8,fp8,0,0.06203199923038483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,64,0,1,float16,fp8,0,0.05056533217430115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,64,0,1,fp8,fp8,0,0.062165334820747375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,64,128,1,float16,float16,0,0.030949334303538006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,64,0,1,float16,float16,0,0.032287999987602234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,64,128,1,float16,fp8,0,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,64,128,1,fp8,fp8,0,0.038506666819254555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,64,0,1,float16,fp8,0,0.030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,64,0,1,fp8,fp8,0,0.03915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,64,128,1,float16,float16,0,0.028229333460330963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,64,0,1,float16,float16,0,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,64,128,1,float16,fp8,0,0.02807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,64,128,1,fp8,fp8,0,0.035173334181308746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,64,0,1,float16,fp8,0,0.028581333657105763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,64,0,1,fp8,fp8,0,0.03491200009981791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,64,128,1,float16,float16,0,0.02861333390076955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,64,128,1,float16,float16,0,0.02978666623433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,64,0,1,float16,float16,0,0.029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,64,0,1,float16,float16,0,0.02890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,64,128,1,float16,fp8,0,0.02874133239189784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,64,128,1,fp8,fp8,0,0.03598399957021078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,64,0,1,fp8,fp8,0,0.03707200040419897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,64,0,1,float16,fp8,0,0.028384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,64,0,1,fp8,fp8,0,0.03566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,64,128,1,float16,fp8,0,0.02978666623433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,64,128,1,fp8,fp8,0,0.03688533355792364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,64,0,1,float16,fp8,0,0.029680001238981884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,64,0,1,float16,fp8,0,0.030666666726271313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,64,128,1,float16,float16,0,0.029578665892283123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,64,0,1,float16,float16,0,0.029951999584833782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,64,128,1,float16,fp8,0,0.03012266755104065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,64,128,1,fp8,fp8,0,0.03706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,64,0,1,fp8,fp8,0,0.037477334340413414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,64,128,1,float16,float16,0,0.023898666103680927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,64,0,1,float16,float16,0,0.024256000916163128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,64,128,1,float16,fp8,0,0.024112001061439514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,64,128,1,fp8,fp8,0,0.028959999481836956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,64,0,1,float16,fp8,0,0.02439466615517934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,64,128,1,fp8,fp8,0,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,64,0,1,fp8,fp8,0,0.028677334388097126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,64,0,1,fp8,fp8,0,0.027999999622503918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,64,128,1,float16,float16,0,0.022053333620230358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,64,0,1,float16,float16,0,0.022229333718617756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,64,128,1,float16,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,64,0,1,float16,fp8,0,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,64,128,1,float16,float16,0,0.02248000105222066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,64,0,1,float16,float16,0,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,64,128,1,float16,fp8,0,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,64,128,1,fp8,fp8,0,0.027823999524116516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,64,0,1,float16,fp8,0,0.02276266614596049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,64,0,1,fp8,fp8,0,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,64,128,1,float16,float16,0,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,64,0,1,float16,float16,0,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,64,128,1,float16,fp8,0,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,64,128,1,fp8,fp8,0,0.028384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,64,0,1,float16,fp8,0,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,64,0,1,fp8,fp8,0,0.028143999477227528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,64,128,1,float16,float16,0,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,64,0,1,float16,float16,0,0.023589332898457844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,64,128,1,float16,fp8,0,0.02350933353106181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,64,128,1,fp8,fp8,0,0.028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,64,128,1,float16,fp8,0,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,64,0,1,float16,fp8,0,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,64,0,1,fp8,fp8,0,0.028602667152881622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,64,128,1,float16,float16,0,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,64,0,1,float16,float16,0,0.01653333380818367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,64,128,1,fp8,fp8,0,0.020629333953062694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,64,0,1,float16,fp8,0,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,64,0,1,fp8,fp8,0,0.020848001043001812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,64,128,1,float16,float16,0,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,64,0,1,float16,float16,0,0.016085332880417507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,64,128,1,float16,fp8,0,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,64,128,1,fp8,fp8,0,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,64,0,1,float16,fp8,0,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,64,0,1,fp8,fp8,0,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,64,128,1,float16,float16,0,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,64,0,1,float16,float16,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,64,128,1,float16,fp8,0,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,64,128,1,fp8,fp8,0,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,64,0,1,float16,fp8,0,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,64,0,1,fp8,fp8,0,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,64,0,1,fp8,fp8,0,0.01951466624935468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,64,128,1,float16,float16,0,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,64,0,1,float16,float16,0,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,64,128,1,float16,fp8,0,0.016528000434239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,64,128,1,fp8,fp8,0,0.020245333512624104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,64,0,1,float16,fp8,0,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,64,128,1,float16,float16,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,64,0,1,float16,float16,0,0.0161013330022494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,64,128,1,float16,fp8,0,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,64,128,1,fp8,fp8,0,0.019589333484570186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,64,0,1,float16,fp8,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,64,0,1,fp8,fp8,0,0.019797333826621372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,64,128,1,float16,float16,0,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,64,0,1,float16,float16,0,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,64,128,1,float16,fp8,0,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,64,128,1,fp8,fp8,0,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,64,0,1,float16,fp8,0,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,64,0,1,fp8,fp8,0,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,64,128,1,float16,float16,0,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,64,128,1,float16,float16,0,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,64,0,1,float16,float16,0,0.013829333086808523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,64,128,1,float16,fp8,0,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,64,128,1,fp8,fp8,0,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,64,0,1,float16,fp8,0,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,64,0,1,fp8,fp8,0,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,64,0,1,float16,float16,0,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,64,128,1,float16,fp8,0,0.014394666999578476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,64,128,1,float16,fp8,0,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,64,128,1,fp8,fp8,0,0.018485333770513535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,64,0,1,float16,fp8,0,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,64,0,1,fp8,fp8,0,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,64,128,1,float16,float16,0,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,64,0,1,float16,float16,0,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,64,128,1,fp8,fp8,0,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,64,0,1,float16,fp8,0,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,64,0,1,fp8,fp8,0,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,64,128,1,float16,float16,0,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,64,0,1,float16,float16,0,0.01423466702302297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,64,128,1,float16,fp8,0,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,64,128,1,fp8,fp8,0,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,64,0,1,float16,fp8,0,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,64,0,1,fp8,fp8,0,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,64,128,1,float16,float16,0,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,64,0,1,float16,float16,0,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,64,128,1,float16,fp8,0,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,64,128,1,fp8,fp8,0,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,64,128,1,float16,fp8,0,0.013631999492645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,64,0,1,float16,fp8,0,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,64,0,1,fp8,fp8,0,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,64,128,1,float16,float16,0,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,64,0,1,float16,float16,0,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,64,128,1,fp8,fp8,0,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,64,0,1,float16,fp8,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,64,0,1,fp8,fp8,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,64,0,1,float16,fp8,0,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,64,128,1,float16,float16,0,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,64,0,1,fp8,fp8,0,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,64,0,1,float16,float16,0,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,64,128,1,float16,fp8,0,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,64,128,1,fp8,fp8,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,64,128,1,float16,float16,0,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,64,0,1,float16,float16,0,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,64,128,1,float16,fp8,0,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,64,128,1,fp8,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,64,0,1,float16,fp8,0,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,64,0,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,64,128,1,float16,float16,0,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,64,0,1,fp8,fp8,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,64,0,1,float16,float16,0,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,64,128,1,float16,fp8,0,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,64,128,1,fp8,fp8,0,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,64,0,1,float16,fp8,0,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,64,128,1,float16,float16,0,0.13827199737230936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,64,0,1,float16,float16,0,0.13828800121943155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,64,0,1,float16,fp8,0,0.13870933651924133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,64,128,1,float16,fp8,0,0.1393173336982727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,64,128,1,fp8,fp8,0,0.23454932371775308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,64,0,1,fp8,fp8,0,0.2358506719271342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,64,128,1,float16,float16,0,0.13921067118644714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,64,128,1,float16,fp8,0,0.13886400063832602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,64,0,1,float16,float16,0,0.13922666509946188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,64,128,1,fp8,fp8,0,0.23547200361887613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,64,0,1,float16,fp8,0,0.1394613285859426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,64,0,1,fp8,fp8,0,0.23605332771937051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,64,128,1,float16,float16,0,0.14470400412877402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,64,0,1,float16,float16,0,0.14469866951306662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,64,128,1,float16,fp8,0,0.14479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,64,128,1,fp8,fp8,0,0.25092800458272296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,64,0,1,float16,fp8,0,0.14455466469128928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,64,0,1,fp8,fp8,0,0.25255467494328815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,64,128,1,float16,float16,0,0.1486186683177948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,64,0,1,float16,float16,0,0.14893333117167154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,64,128,1,float16,fp8,0,0.14788267016410828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,64,128,1,fp8,fp8,0,0.2494986653327942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,64,0,1,float16,fp8,0,0.14813866217931113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,64,128,1,float16,float16,0,0.1097920040289561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,64,0,1,fp8,fp8,0,0.2490239938100179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,64,128,1,fp8,fp8,0,0.14333333571751913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,64,0,1,float16,float16,0,0.10968533158302307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,64,128,1,float16,fp8,0,0.10583999752998352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,64,0,1,float16,fp8,0,0.10651733477910359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,64,0,1,fp8,fp8,0,0.1426400045553843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,64,128,1,float16,float16,0,0.07690666615962982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,64,0,1,float16,float16,0,0.07713066538174947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,64,128,1,float16,fp8,0,0.07739200194676717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,64,128,1,fp8,fp8,0,0.1293653349081675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,64,128,1,float16,fp8,0,0.07807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,64,0,1,float16,fp8,0,0.0767626663049062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,64,0,1,fp8,fp8,0,0.13089600205421448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,64,128,1,float16,float16,0,0.07937066753705342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,64,0,1,float16,float16,0,0.07842133442560832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,64,0,1,float16,fp8,0,0.0775626649459203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,64,0,1,float16,float16,0,0.082096000512441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,64,128,1,fp8,fp8,0,0.1295413374900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,64,128,1,float16,float16,0,0.08304533362388611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,64,0,1,fp8,fp8,0,0.12948800126711527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,64,128,1,float16,fp8,0,0.08274133503437042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,64,128,1,fp8,fp8,0,0.13674666484196982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,64,0,1,float16,fp8,0,0.08281599978605907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,64,128,1,float16,float16,0,0.08498666683832805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,64,0,1,fp8,fp8,0,0.13795199990272522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,64,0,1,float16,float16,0,0.08463467160860698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,64,128,1,float16,fp8,0,0.0839573343594869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,64,128,1,fp8,fp8,0,0.13587199648221335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,64,0,1,float16,fp8,0,0.0846613347530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,64,0,1,fp8,fp8,0,0.13642666737238565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,64,128,1,float16,float16,0,0.06029333174228668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,64,0,1,float16,float16,0,0.06081599990526835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,64,128,1,float16,fp8,0,0.057706668972969055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,64,128,1,fp8,fp8,0,0.08218133449554443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,64,0,1,float16,fp8,0,0.05787200232346853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,64,0,1,fp8,fp8,0,0.08163733283678691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,64,128,1,float16,float16,0,0.04378133515516917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,64,0,1,float16,float16,0,0.04394666850566864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,64,128,1,float16,fp8,0,0.04354133208592733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,64,128,1,fp8,fp8,0,0.07268799841403961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,64,128,1,fp8,fp8,0,0.0735093355178833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,64,0,1,float16,fp8,0,0.04332800209522247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,64,0,1,fp8,fp8,0,0.07190399865309398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,64,128,1,float16,float16,0,0.04445866743723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,64,0,1,float16,float16,0,0.044906665881474815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,64,128,1,float16,fp8,0,0.04428266485532125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,64,0,1,float16,fp8,0,0.04398400088151296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,64,0,1,fp8,fp8,0,0.0730560024579366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,64,0,1,float16,fp8,0,0.046367997924486794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,64,128,1,float16,float16,0,0.046351999044418335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,64,0,1,float16,float16,0,0.04570133487383524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,64,128,1,float16,fp8,0,0.04619733492533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,64,128,1,fp8,fp8,0,0.07638399799664815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,64,0,1,fp8,fp8,0,0.07699733475844066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,64,128,1,float16,float16,0,0.04725333551565806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,64,0,1,float16,float16,0,0.0473280002673467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,64,128,1,float16,fp8,0,0.047370667258898415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,64,128,1,fp8,fp8,0,0.07734400033950806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,64,0,1,float16,fp8,0,0.047482664386431374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,64,128,1,float16,float16,0,0.029552000264326733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,64,0,1,fp8,fp8,0,0.07634666562080383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,64,0,1,float16,float16,0,0.030042665700117748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,64,128,1,float16,fp8,0,0.02956266701221466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,64,128,1,fp8,fp8,0,0.046522667010625206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,64,0,1,float16,fp8,0,0.029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,64,0,1,fp8,fp8,0,0.04715733230113983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,64,128,1,float16,float16,0,0.026464000344276428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,64,0,1,float16,float16,0,0.02628266563018163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,64,128,1,float16,fp8,0,0.026672000686327618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,64,128,1,fp8,fp8,0,0.04322666426499685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,64,0,1,float16,fp8,0,0.026687999566396076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,64,0,1,fp8,fp8,0,0.04307200014591217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,64,128,1,float16,float16,0,0.02682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,64,0,1,float16,float16,0,0.027087998886903126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,64,128,1,float16,fp8,0,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,64,128,1,fp8,fp8,0,0.04317333300908407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,64,0,1,float16,fp8,0,0.026565333207448322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,64,0,1,fp8,fp8,0,0.04337066908677419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,64,128,1,float16,float16,0,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,64,0,1,float16,float16,0,0.027530667682488758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,64,128,1,float16,fp8,0,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,64,128,1,fp8,fp8,0,0.044533332188924156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,64,0,1,float16,fp8,0,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,64,0,1,fp8,fp8,0,0.04455466568470001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,64,128,1,float16,float16,0,0.027952000498771667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,64,0,1,float16,float16,0,0.028586665789286297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,64,128,1,float16,fp8,0,0.02787200113137563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,64,128,1,fp8,fp8,0,0.04394666850566864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,64,0,1,float16,fp8,0,0.027973333994547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,64,0,1,fp8,fp8,0,0.04385066529115041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,64,128,1,float16,float16,0,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,64,0,1,float16,float16,0,0.02195200075705846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,64,128,1,float16,fp8,0,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,64,128,1,fp8,fp8,0,0.031066666046778362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,64,0,1,float16,fp8,0,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,64,0,1,fp8,fp8,0,0.031248000760873158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,64,0,1,float16,fp8,0,0.020618667205174763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,64,128,1,float16,float16,0,0.020266667008399963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,64,0,1,fp8,fp8,0,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,64,0,1,float16,float16,0,0.020495999604463577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,64,128,1,float16,fp8,0,0.02067733307679494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,64,128,1,fp8,fp8,0,0.029834667841593426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,64,128,1,float16,float16,0,0.020351999749739964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,64,0,1,float16,float16,0,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,64,128,1,float16,fp8,0,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,64,128,1,fp8,fp8,0,0.029802667597929638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,64,0,1,float16,fp8,0,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,64,0,1,fp8,fp8,0,0.030389333764712017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,64,128,1,float16,float16,0,0.02089600016673406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,64,0,1,float16,float16,0,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,64,128,1,float16,fp8,0,0.021856000026067097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,64,128,1,fp8,fp8,0,0.030778666337331135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,64,0,1,float16,fp8,0,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,64,0,1,fp8,fp8,0,0.03030399978160858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,64,128,1,float16,float16,0,0.021733333667119343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,64,0,1,float16,float16,0,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,64,128,1,float16,fp8,0,0.021477334201335907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,64,128,1,fp8,fp8,0,0.031013332307338715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,64,0,1,float16,fp8,0,0.02162666618824005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,64,0,1,fp8,fp8,0,0.030997333427270252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,64,128,1,float16,float16,0,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,64,0,1,float16,float16,0,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,64,128,1,float16,fp8,0,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,64,128,1,fp8,fp8,0,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,64,0,1,float16,fp8,0,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,64,0,1,fp8,fp8,0,0.023706667125225067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,64,128,1,float16,float16,0,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,64,0,1,float16,float16,0,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,64,128,1,float16,fp8,0,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,64,128,1,fp8,fp8,0,0.022128000855445862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,64,0,1,float16,fp8,0,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,64,0,1,fp8,fp8,0,0.022634667654832203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,64,128,1,float16,float16,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,64,0,1,float16,float16,0,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,64,128,1,float16,fp8,0,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,64,128,1,float16,fp8,0,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,64,128,1,fp8,fp8,0,0.023007998863856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,64,0,1,float16,fp8,0,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,64,0,1,fp8,fp8,0,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,64,128,1,float16,float16,0,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,64,0,1,float16,float16,0,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,64,128,1,float16,fp8,0,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,64,128,1,fp8,fp8,0,0.022687998910744984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,64,0,1,float16,fp8,0,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,64,0,1,fp8,fp8,0,0.0229120006163915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,64,128,1,float16,float16,0,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,64,0,1,float16,float16,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,64,128,1,fp8,fp8,0,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,64,0,1,float16,fp8,0,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,64,0,1,fp8,fp8,0,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,64,128,1,float16,float16,0,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,64,0,1,float16,float16,0,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,64,128,1,float16,fp8,0,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,64,128,1,fp8,fp8,0,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,64,0,1,float16,fp8,0,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,64,0,1,fp8,fp8,0,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,64,128,1,float16,float16,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,64,0,1,float16,float16,0,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,64,128,1,float16,fp8,0,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,64,0,1,float16,fp8,0,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,64,128,1,fp8,fp8,0,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,64,0,1,fp8,fp8,0,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,64,128,1,float16,float16,0,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,64,0,1,fp8,fp8,0,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,64,0,1,float16,float16,0,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,64,128,1,float16,fp8,0,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,64,128,1,fp8,fp8,0,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,64,0,1,float16,fp8,0,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,64,128,1,float16,float16,0,0.013482666263977686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,64,0,1,fp8,fp8,0,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,64,0,1,float16,float16,0,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,64,128,1,float16,float16,0,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,64,128,1,float16,fp8,0,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,64,128,1,fp8,fp8,0,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,64,0,1,float16,fp8,0,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,64,0,1,float16,float16,0,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,64,128,1,float16,fp8,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,64,128,1,fp8,fp8,0,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,64,0,1,float16,fp8,0,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,64,0,1,fp8,fp8,0,0.01993600030740102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,64,128,1,float16,float16,0,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,64,0,1,float16,float16,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,64,128,1,float16,fp8,0,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,64,128,1,fp8,fp8,0,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,64,0,1,float16,fp8,0,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,64,0,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,64,128,1,float16,float16,0,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,64,0,1,float16,float16,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,64,128,1,float16,fp8,0,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,64,128,1,fp8,fp8,0,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,64,0,1,float16,fp8,0,0.01257066677014033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,64,0,1,fp8,fp8,0,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,64,128,1,float16,float16,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,64,0,1,float16,float16,0,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,64,128,1,float16,fp8,0,0.012565333396196365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,64,128,1,fp8,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,64,0,1,float16,fp8,0,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,64,0,1,fp8,fp8,0,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,64,0,1,fp8,fp8,0,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,64,128,1,float16,float16,0,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,64,0,1,float16,float16,0,0.012266666938861212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,64,128,1,float16,fp8,0,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,64,128,1,fp8,fp8,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,64,0,1,float16,fp8,0,0.012517333030700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,64,128,1,float16,float16,0,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,64,0,1,float16,float16,0,0.011930666863918304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,64,128,1,float16,fp8,0,0.012565333396196365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,64,128,1,fp8,fp8,0,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,64,0,1,float16,fp8,0,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,64,0,1,fp8,fp8,0,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,64,128,1,float16,float16,0,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,64,0,1,float16,float16,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,64,128,1,float16,fp8,0,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,64,128,1,fp8,fp8,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,64,0,1,float16,fp8,0,0.012159999459981918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,64,0,1,fp8,fp8,0,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,64,128,1,float16,float16,0,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,64,0,1,float16,float16,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,64,128,1,float16,fp8,0,0.012085333466529846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,64,128,1,fp8,fp8,0,0.017770666629076004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,64,0,1,float16,fp8,0,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,64,0,1,fp8,fp8,0,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,64,128,1,float16,float16,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,64,0,1,float16,float16,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,64,128,1,float16,fp8,0,0.012378666549921036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,64,128,1,fp8,fp8,0,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,64,0,1,float16,fp8,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,64,0,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,64,128,1,float16,float16,0,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,64,0,1,float16,float16,0,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,64,128,1,float16,fp8,0,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,64,128,1,fp8,fp8,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,64,0,1,float16,fp8,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,64,0,1,fp8,fp8,0,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,64,128,1,float16,float16,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,64,0,1,float16,float16,0,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,64,128,1,float16,float16,0,0.08349333206812541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,64,128,1,float16,fp8,0,0.012400000045696894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,64,0,1,float16,float16,0,0.08310399949550629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,64,128,1,fp8,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,64,0,1,float16,fp8,0,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,64,0,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,64,128,1,float16,float16,0,0.08457600076993306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,64,128,1,float16,fp8,0,0.08282666901747386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,64,0,1,float16,fp8,0,0.08294400076071422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,64,128,1,fp8,fp8,0,0.1982133388519287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,64,0,1,fp8,fp8,0,0.19803200165430704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,64,0,1,float16,float16,0,0.0846666693687439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,64,128,1,float16,fp8,0,0.08362666765848796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,64,0,1,float16,fp8,0,0.08418132861455281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,64,128,1,fp8,fp8,0,0.19826666514078775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,64,0,1,fp8,fp8,0,0.19951999187469482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,64,128,1,fp8,fp8,0,0.2059146761894226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,64,128,1,float16,float16,0,0.09084266424179077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,64,0,1,float16,float16,0,0.09034666419029236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,64,128,1,float16,fp8,0,0.09061333537101746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,64,0,1,float16,fp8,0,0.09021332859992981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,64,0,1,fp8,fp8,0,0.20643732945124307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,64,128,1,float16,float16,0,0.09313066800435384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,64,0,1,float16,float16,0,0.09313600262006123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,64,128,1,float16,fp8,0,0.09239466985066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,64,0,1,float16,fp8,0,0.09242666761080424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,64,128,1,fp8,fp8,0,0.2052746613820394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,64,128,1,float16,float16,0,0.060933331648508705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,64,0,1,float16,float16,0,0.06099733213583628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,64,0,1,fp8,fp8,0,0.20588799317677817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,64,128,1,float16,fp8,0,0.05844266712665558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,64,128,1,fp8,fp8,0,0.11769066254297893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,64,0,1,float16,fp8,0,0.058058664202690125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,64,0,1,fp8,fp8,0,0.1170186698436737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,64,128,1,float16,float16,0,0.04614933331807455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,64,0,1,float16,float16,0,0.045567999283472695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,64,128,1,float16,fp8,0,0.04609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,64,128,1,float16,float16,0,0.04641066491603851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,64,128,1,fp8,fp8,0,0.10778133074442546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,64,0,1,float16,fp8,0,0.04571733375390371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,64,0,1,fp8,fp8,0,0.10737066467603047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,64,0,1,float16,float16,0,0.04597333570321401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,64,128,1,float16,fp8,0,0.04580800235271454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,64,0,1,float16,fp8,0,0.045519997676213585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,64,128,1,fp8,fp8,0,0.10810133814811707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,64,0,1,fp8,fp8,0,0.10817066828409831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,64,128,1,fp8,fp8,0,0.11177600423494975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,64,0,1,float16,fp8,0,0.04781333108743032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,64,128,1,float16,float16,0,0.048341333866119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,64,0,1,float16,float16,0,0.047498668233553566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,64,128,1,float16,fp8,0,0.04756266872088114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,64,0,1,fp8,fp8,0,0.11157332857449849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,64,128,1,float16,float16,0,0.04947733382383982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,64,0,1,float16,float16,0,0.04902400076389313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,64,128,1,float16,float16,0,0.030037333567937214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,64,128,1,float16,fp8,0,0.04894933104515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,64,128,1,fp8,fp8,0,0.11204266548156738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,64,0,1,float16,fp8,0,0.049178664882977806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,64,0,1,fp8,fp8,0,0.11205866932868958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,64,128,1,float16,float16,0,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,64,0,1,float16,float16,0,0.030421334008375805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,64,0,1,float16,float16,0,0.02757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,64,128,1,float16,fp8,0,0.030074665943781536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,64,128,1,fp8,fp8,0,0.0662666658560435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,64,0,1,float16,fp8,0,0.02940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,64,0,1,fp8,fp8,0,0.06658666829268138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,64,128,1,float16,fp8,0,0.027503999571005504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,64,128,1,fp8,fp8,0,0.062496001521746315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,64,0,1,float16,fp8,0,0.027503999571005504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,64,0,1,fp8,fp8,0,0.06241066753864288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,64,128,1,float16,float16,0,0.027690666417280834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,64,0,1,float16,float16,0,0.028005334238211315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,64,128,1,float16,fp8,0,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,64,128,1,fp8,fp8,0,0.0624533345301946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,64,0,1,float16,fp8,0,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,64,0,1,fp8,fp8,0,0.06115200122197469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,64,128,1,float16,float16,0,0.028346667687098186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,64,0,1,float16,float16,0,0.028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,64,128,1,float16,fp8,0,0.028501334289709728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,64,128,1,fp8,fp8,0,0.06448000172773997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,64,0,1,float16,fp8,0,0.028389332195123036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,64,0,1,fp8,fp8,0,0.06384533147017162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,64,128,1,float16,float16,0,0.03044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,64,0,1,float16,float16,0,0.02900800108909607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,64,128,1,float16,fp8,0,0.028922667105992634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,64,128,1,fp8,fp8,0,0.06399466594060262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,64,0,1,float16,fp8,0,0.02902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,64,0,1,fp8,fp8,0,0.06407466530799866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,64,128,1,float16,float16,0,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,64,0,1,float16,float16,0,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,64,128,1,float16,fp8,0,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,64,128,1,fp8,fp8,0,0.04083733260631561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,64,0,1,float16,fp8,0,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,64,0,1,fp8,fp8,0,0.04045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,64,128,1,float16,float16,0,0.019823999454577763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,64,0,1,float16,float16,0,0.019653332730134327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,64,128,1,float16,fp8,0,0.019866666446129482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,64,128,1,fp8,fp8,0,0.04022933294375738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,64,0,1,float16,fp8,0,0.01985599969824155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,64,0,1,fp8,fp8,0,0.03947199881076813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,64,128,1,float16,float16,0,0.02019199977318446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,64,0,1,float16,float16,0,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,64,0,1,float16,float16,0,0.02059200033545494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,64,128,1,float16,fp8,0,0.020069333414236706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,64,128,1,fp8,fp8,0,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,64,0,1,float16,fp8,0,0.020154666155576706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,64,0,1,fp8,fp8,0,0.03973866750796636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,64,128,1,float16,float16,0,0.020394666741291683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,64,128,1,float16,fp8,0,0.02073066681623459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,64,128,1,float16,fp8,0,0.020693333198626835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,64,128,1,fp8,fp8,0,0.0400693342089653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,64,128,1,fp8,fp8,0,0.04080000023047129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,64,0,1,float16,fp8,0,0.02082666630546252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,64,0,1,float16,fp8,0,0.020400000115235645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,64,0,1,fp8,fp8,0,0.04041599979003271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,64,128,1,float16,float16,0,0.020666666328907013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,64,0,1,float16,float16,0,0.020538666596015293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,64,0,1,fp8,fp8,0,0.040063999593257904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,64,128,1,float16,float16,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,64,0,1,fp8,fp8,0,0.02649066597223282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,64,0,1,float16,float16,0,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,64,128,1,float16,fp8,0,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,64,128,1,fp8,fp8,0,0.026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,64,0,1,float16,fp8,0,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,64,128,1,float16,float16,0,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,64,0,1,float16,float16,0,0.014357333381970724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,64,128,1,float16,fp8,0,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,64,128,1,fp8,fp8,0,0.0262773334980011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,64,0,1,float16,fp8,0,0.014309333016475042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,64,0,1,fp8,fp8,0,0.026074667771657307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,64,128,1,float16,float16,0,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,64,0,1,float16,float16,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,64,128,1,float16,fp8,0,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,64,0,1,float16,float16,0,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,64,128,1,fp8,fp8,0,0.026165333886941273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,64,0,1,float16,fp8,0,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,64,0,1,fp8,fp8,0,0.026213333010673523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,64,0,1,fp8,fp8,0,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,64,128,1,float16,float16,0,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,64,128,1,float16,fp8,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,64,128,1,fp8,fp8,0,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,64,0,1,float16,fp8,0,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,64,128,1,float16,float16,0,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,64,0,1,float16,float16,0,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,64,128,1,float16,fp8,0,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,64,128,1,fp8,fp8,0,0.026496000587940216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,64,0,1,float16,fp8,0,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,64,0,1,fp8,fp8,0,0.026389333109060924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,64,128,1,float16,float16,0,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,64,0,1,fp8,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,64,0,1,float16,float16,0,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,64,128,1,float16,fp8,0,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,64,128,1,fp8,fp8,0,0.022298666338125866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,64,0,1,float16,fp8,0,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,64,0,1,float16,fp8,0,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,64,128,1,float16,float16,0,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,64,0,1,float16,float16,0,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,64,128,1,float16,fp8,0,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,64,128,1,fp8,fp8,0,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,64,0,1,fp8,fp8,0,0.02197866638501485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,64,128,1,float16,float16,0,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,64,0,1,float16,float16,0,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,64,128,1,float16,fp8,0,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,64,128,1,fp8,fp8,0,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,64,128,1,fp8,fp8,0,0.022064000368118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,64,0,1,fp8,fp8,0,0.021935999393463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,64,128,1,float16,float16,0,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,64,0,1,float16,fp8,0,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,64,128,1,float16,float16,0,0.012666666259368261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,64,0,1,float16,float16,0,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,64,0,1,float16,fp8,0,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,64,128,1,float16,fp8,0,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,64,0,1,float16,fp8,0,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,64,0,1,fp8,fp8,0,0.02183466653029124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,64,0,1,float16,float16,0,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,64,128,1,float16,fp8,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,64,128,1,fp8,fp8,0,0.022490667800108593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,64,0,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,64,0,1,fp8,fp8,0,0.022261333962281544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,64,128,1,float16,float16,0,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,64,0,1,float16,float16,0,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,64,128,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,64,128,1,fp8,fp8,0,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,64,128,1,fp8,fp8,0,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,64,0,1,fp8,fp8,0,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,64,0,1,fp8,fp8,0,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,64,128,1,float16,float16,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,64,0,1,float16,float16,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,64,128,1,float16,fp8,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,64,0,1,float16,fp8,0,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,64,128,1,float16,float16,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,64,0,1,float16,float16,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,64,128,1,float16,fp8,0,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,64,128,1,fp8,fp8,0,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,64,0,1,float16,fp8,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,64,0,1,fp8,fp8,0,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,64,128,1,float16,float16,0,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,64,0,1,float16,float16,0,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,64,128,1,float16,fp8,0,0.012202666451533636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,64,128,1,float16,fp8,0,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,64,128,1,fp8,fp8,0,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,64,128,1,fp8,fp8,0,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,64,0,1,float16,fp8,0,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,64,0,1,fp8,fp8,0,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,64,128,1,float16,float16,0,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,64,0,1,float16,float16,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,64,0,1,float16,fp8,0,0.012256000190973282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,64,0,1,fp8,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,64,128,1,float16,float16,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,64,0,1,float16,float16,0,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,64,128,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,64,128,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,64,0,1,float16,fp8,0,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,64,0,1,fp8,fp8,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,64,128,1,float16,float16,0,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,64,0,1,float16,float16,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,64,128,1,float16,fp8,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,64,128,1,fp8,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,64,0,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,64,0,1,fp8,fp8,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,64,128,1,float16,float16,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,64,0,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,64,128,1,float16,fp8,0,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,64,128,1,fp8,fp8,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,64,0,1,float16,fp8,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,64,0,1,fp8,fp8,0,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,64,128,1,float16,float16,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,64,0,1,float16,float16,0,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,64,128,1,float16,fp8,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,64,128,1,fp8,fp8,0,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,64,0,1,float16,fp8,0,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,64,0,1,fp8,fp8,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,64,128,1,float16,float16,0,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,64,128,1,float16,float16,0,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,64,0,1,float16,float16,0,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,64,128,1,float16,fp8,0,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,64,128,1,fp8,fp8,0,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,64,0,1,fp8,fp8,0,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,64,128,1,float16,fp8,0,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,64,0,1,float16,float16,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,64,128,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,64,0,1,float16,fp8,0,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,64,0,1,fp8,fp8,0,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,64,0,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,64,128,1,float16,float16,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,64,0,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,64,128,1,float16,fp8,0,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,64,128,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,64,0,1,float16,fp8,0,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,64,128,1,float16,float16,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,64,0,1,float16,float16,0,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,64,128,1,float16,fp8,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,64,128,1,fp8,fp8,0,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,64,0,1,float16,fp8,0,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,64,0,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,64,128,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,64,0,1,float16,float16,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,64,128,1,float16,fp8,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,64,128,1,fp8,fp8,0,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,64,0,1,float16,fp8,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,64,0,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,64,128,1,float16,float16,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,64,0,1,float16,float16,0,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,64,128,1,float16,fp8,0,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,64,128,1,fp8,fp8,0,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,64,0,1,float16,fp8,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,64,128,1,fp8,fp8,0,0.17615467309951782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,64,0,1,fp8,fp8,0,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,64,128,1,float16,float16,0,0.06001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,64,0,1,float16,float16,0,0.06039999922116598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,64,128,1,float16,fp8,0,0.06029866635799408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,64,0,1,float16,fp8,0,0.05989866455396017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,64,0,1,fp8,fp8,0,0.17602133750915527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,64,128,1,float16,float16,0,0.06109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,64,0,1,float16,float16,0,0.060549333691596985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,64,128,1,float16,fp8,0,0.05990933378537496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,64,0,1,float16,fp8,0,0.06025066475073496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,64,128,1,fp8,fp8,0,0.17775466044743857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,64,0,1,fp8,fp8,0,0.17668799559275308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,64,128,1,float16,float16,0,0.0614026685555776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,64,0,1,float16,float16,0,0.06155733267466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,64,128,1,float16,fp8,0,0.062208001812299095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,64,128,1,fp8,fp8,0,0.18127467234929404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,64,0,1,float16,fp8,0,0.06196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,64,0,1,fp8,fp8,0,0.18117332458496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,64,128,1,float16,float16,0,0.06348266700903575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,64,0,1,float16,float16,0,0.06322133541107178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,64,128,1,float16,fp8,0,0.06328533093134563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,64,0,1,float16,fp8,0,0.06302399933338165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,64,128,1,fp8,fp8,0,0.18132799863815308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,64,128,1,float16,float16,0,0.03569599986076355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,64,0,1,fp8,fp8,0,0.18127467234929404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,64,0,1,float16,float16,0,0.036042665441830955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,64,128,1,float16,fp8,0,0.03579733272393545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,64,128,1,fp8,fp8,0,0.10033599535624187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,64,0,1,float16,fp8,0,0.03504000107447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,64,0,1,fp8,fp8,0,0.10086400310198466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,64,128,1,float16,float16,0,0.03446933378775915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,64,0,1,float16,float16,0,0.034501334031422935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,64,128,1,float16,fp8,0,0.03460799902677536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,64,128,1,fp8,fp8,0,0.0962720016638438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,64,0,1,float16,fp8,0,0.03487999985615412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,64,128,1,fp8,fp8,0,0.09653866291046143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,64,0,1,fp8,fp8,0,0.09620799620946248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,64,128,1,float16,float16,0,0.03477333237727483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,64,0,1,float16,float16,0,0.03482133398453394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,64,128,1,float16,fp8,0,0.03499199946721395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,64,0,1,float16,fp8,0,0.03495466709136963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,64,0,1,fp8,fp8,0,0.09693333506584167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,64,128,1,float16,float16,0,0.03565866748491923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,64,0,1,float16,float16,0,0.035690667728583016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,64,0,1,fp8,fp8,0,0.09758399923642476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,64,128,1,float16,fp8,0,0.03584533433119456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,64,128,1,fp8,fp8,0,0.09731733798980713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,64,0,1,float16,fp8,0,0.03537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,64,128,1,float16,float16,0,0.036362667878468834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,64,0,1,float16,float16,0,0.03606933355331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,64,128,1,float16,fp8,0,0.036144000788529716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,64,128,1,fp8,fp8,0,0.09749333063761394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,64,0,1,float16,fp8,0,0.036346666514873505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,64,128,1,float16,fp8,0,0.023925334215164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,64,0,1,fp8,fp8,0,0.09726933638254802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,64,128,1,float16,float16,0,0.02385599911212921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,64,0,1,float16,float16,0,0.0242399995525678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,64,128,1,fp8,fp8,0,0.05874133110046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,64,0,1,float16,fp8,0,0.024330665667851765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,64,0,1,fp8,fp8,0,0.05886933207511902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,64,128,1,float16,float16,0,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,64,0,1,fp8,fp8,0,0.058378666639328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,64,0,1,float16,float16,0,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,64,128,1,float16,fp8,0,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,64,128,1,fp8,fp8,0,0.05771199862162272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,64,0,1,float16,fp8,0,0.023845332364241283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,64,128,1,float16,float16,0,0.02386133372783661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,64,0,1,float16,float16,0,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,64,128,1,float16,fp8,0,0.02386666586001714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,64,128,1,fp8,fp8,0,0.05763733386993408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,64,0,1,float16,fp8,0,0.024160000185171764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,64,0,1,fp8,fp8,0,0.05845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,64,128,1,float16,float16,0,0.02436800052722295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,64,0,1,float16,float16,0,0.02418133368094762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,64,128,1,float16,fp8,0,0.02443733314673106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,64,128,1,fp8,fp8,0,0.059418668349583946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,64,0,1,float16,fp8,0,0.024341332415739696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,64,0,1,fp8,fp8,0,0.05881600081920624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,64,128,1,float16,float16,0,0.024447999894618988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,64,0,1,float16,float16,0,0.02439466615517934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,64,0,1,fp8,fp8,0,0.0588319996992747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,64,128,1,float16,fp8,0,0.02460266649723053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,64,128,1,fp8,fp8,0,0.05979733169078827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,64,128,1,float16,fp8,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,64,128,1,fp8,fp8,0,0.03585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,64,0,1,float16,fp8,0,0.024618667860825855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,64,128,1,float16,float16,0,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,64,0,1,float16,float16,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,64,0,1,float16,fp8,0,0.01607999950647354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,64,0,1,fp8,fp8,0,0.036229332288106285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,64,128,1,float16,float16,0,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,64,0,1,float16,float16,0,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,64,128,1,float16,fp8,0,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,64,128,1,fp8,fp8,0,0.03583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,64,0,1,float16,fp8,0,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,64,0,1,fp8,fp8,0,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,64,128,1,float16,float16,0,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,64,0,1,float16,float16,0,0.015487999965747198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,64,128,1,float16,fp8,0,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,64,128,1,fp8,fp8,0,0.035232000052928925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,64,0,1,float16,fp8,0,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,64,0,1,fp8,fp8,0,0.03552533437808355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,64,128,1,float16,float16,0,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,64,0,1,float16,float16,0,0.015989333391189575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,64,128,1,float16,fp8,0,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,64,128,1,fp8,fp8,0,0.035605333745479584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,64,0,1,float16,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,64,0,1,fp8,fp8,0,0.03522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,64,128,1,float16,float16,0,0.01599466676513354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,64,0,1,float16,float16,0,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,64,128,1,float16,fp8,0,0.01653333380818367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,64,128,1,fp8,fp8,0,0.03603733330965042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,64,0,1,float16,fp8,0,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,64,0,1,fp8,fp8,0,0.03570666660865148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,64,0,1,float16,fp8,0,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,64,128,1,float16,float16,0,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,64,0,1,float16,float16,0,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,64,128,1,float16,fp8,0,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,64,128,1,fp8,fp8,0,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,64,0,1,fp8,fp8,0,0.02613866577545802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,64,128,1,float16,float16,0,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,64,0,1,float16,float16,0,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,64,128,1,float16,fp8,0,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,64,128,1,fp8,fp8,0,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,64,0,1,float16,fp8,0,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,64,0,1,fp8,fp8,0,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,64,128,1,float16,float16,0,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,64,0,1,float16,float16,0,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,64,128,1,float16,fp8,0,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,64,128,1,fp8,fp8,0,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,64,0,1,float16,fp8,0,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,64,0,1,fp8,fp8,0,0.025472000241279602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,64,0,1,fp8,fp8,0,0.02496533344189326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,64,128,1,float16,float16,0,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,64,128,1,float16,float16,0,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,64,0,1,float16,float16,0,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,64,128,1,float16,fp8,0,0.013482666263977686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,64,128,1,fp8,fp8,0,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,64,0,1,float16,fp8,0,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,64,0,1,float16,float16,0,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,64,128,1,float16,fp8,0,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,64,128,1,fp8,fp8,0,0.025626666843891144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,64,0,1,float16,fp8,0,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,64,128,1,fp8,fp8,0,0.02197866638501485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,64,0,1,fp8,fp8,0,0.02601066728432973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,64,128,1,float16,float16,0,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,64,0,1,float16,float16,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,64,128,1,float16,fp8,0,0.012266666938861212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,64,0,1,float16,fp8,0,0.012298667182525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,64,0,1,fp8,fp8,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,64,128,1,float16,float16,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,64,0,1,float16,float16,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,64,128,1,float16,fp8,0,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,64,128,1,fp8,fp8,0,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,64,0,1,float16,fp8,0,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,64,0,1,fp8,fp8,0,0.02075733368595441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,64,128,1,float16,float16,0,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,64,0,1,float16,float16,0,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,64,128,1,float16,fp8,0,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,64,128,1,fp8,fp8,0,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,64,0,1,float16,fp8,0,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,64,0,1,fp8,fp8,0,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,64,128,1,float16,float16,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,64,0,1,float16,float16,0,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,64,128,1,float16,fp8,0,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,64,128,1,fp8,fp8,0,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,64,0,1,float16,fp8,0,0.012250666817029318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,64,0,1,fp8,fp8,0,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,64,128,1,float16,float16,0,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,64,0,1,float16,float16,0,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,64,128,1,float16,fp8,0,0.012565333396196365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,64,128,1,fp8,fp8,0,0.02089600016673406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,64,0,1,float16,fp8,0,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,64,0,1,fp8,fp8,0,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,64,128,1,float16,float16,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,64,0,1,float16,float16,0,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,64,128,1,float16,fp8,0,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,64,128,1,fp8,fp8,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,64,0,1,float16,fp8,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,64,0,1,fp8,fp8,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,64,128,1,float16,float16,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,64,128,1,float16,fp8,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,64,128,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,64,0,1,float16,fp8,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,64,0,1,fp8,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,64,0,1,fp8,fp8,0,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,64,128,1,float16,float16,0,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,64,0,1,float16,float16,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,64,128,1,float16,float16,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,64,128,1,float16,fp8,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,64,128,1,fp8,fp8,0,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,64,0,1,float16,fp8,0,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,64,0,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,64,128,1,float16,fp8,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,64,128,1,float16,fp8,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,64,128,1,fp8,fp8,0,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,64,0,1,float16,fp8,0,0.011920000116030375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,64,0,1,fp8,fp8,0,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,64,128,1,float16,float16,0,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,64,0,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,64,128,1,float16,float16,0,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,64,0,1,float16,float16,0,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,64,128,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,64,0,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,64,0,1,float16,float16,0,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,64,128,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,64,128,1,fp8,fp8,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,64,0,1,float16,fp8,0,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,64,0,1,float16,fp8,0,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,64,0,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,64,128,1,float16,float16,0,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,64,128,1,float16,float16,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,64,0,1,float16,float16,0,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,64,128,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,64,128,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,64,0,1,fp8,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,64,0,1,float16,float16,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,64,128,1,float16,fp8,0,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,64,128,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,64,0,1,float16,fp8,0,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,64,0,1,fp8,fp8,0,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,64,128,1,float16,float16,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,64,0,1,float16,float16,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,64,0,1,float16,float16,0,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,64,128,1,float16,fp8,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,64,128,1,fp8,fp8,0,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,64,0,1,float16,fp8,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,64,0,1,fp8,fp8,0,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,64,128,1,float16,float16,0,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,64,128,1,float16,fp8,0,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,64,128,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,64,0,1,float16,fp8,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,64,0,1,float16,fp8,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,64,0,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,64,128,1,float16,float16,0,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,64,0,1,float16,float16,0,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,64,128,1,float16,fp8,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,64,128,1,fp8,fp8,0,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,64,0,1,float16,fp8,0,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,64,0,1,fp8,fp8,0,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,64,128,1,float16,float16,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,64,0,1,float16,float16,0,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,64,128,1,float16,fp8,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,64,128,1,fp8,fp8,0,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,64,0,1,fp8,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,64,128,1,float16,float16,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,64,0,1,float16,float16,0,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,64,128,1,float16,fp8,0,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,64,128,1,fp8,fp8,0,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,64,0,1,float16,fp8,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,64,0,1,fp8,fp8,0,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,64,128,1,float16,float16,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,64,0,1,float16,float16,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,64,128,1,float16,fp8,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,64,128,1,fp8,fp8,0,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,64,0,1,float16,fp8,0,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,64,0,1,fp8,fp8,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,64,128,1,float16,float16,0,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,64,0,1,float16,float16,0,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,64,128,1,float16,fp8,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,64,128,1,fp8,fp8,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,64,0,1,float16,fp8,0,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,64,0,1,fp8,fp8,0,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,64,128,1,float16,float16,0,1.4655146598815918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,64,128,1,float16,fp8,0,1.449354648590088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,64,128,1,fp8,fp8,0,1.8729333877563477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,64,128,1,float16,float16,0,1.5354986190795898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,64,128,1,float16,fp8,0,1.5185546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,64,128,1,fp8,fp8,0,1.9363093376159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,64,128,1,float16,float16,0,1.5499146779378254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,64,0,1,fp8,fp8,0,8.598053614298502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,64,128,1,float16,fp8,0,1.5360426902770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,64,0,1,float16,float16,0,9.50221316019694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,64,0,1,float16,fp8,0,9.532906850179037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,64,0,1,float16,float16,0,9.561471939086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,64,128,1,fp8,fp8,0,1.9475733439127605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,64,128,1,float16,float16,0,1.5989227294921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,64,0,1,float16,fp8,0,9.45799446105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,64,0,1,fp8,fp8,0,8.647258758544922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,64,128,1,float16,fp8,0,1.5703946749369304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,64,128,1,fp8,fp8,0,2.0011146863301597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,64,128,1,float16,float16,0,0.8629279931386312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,64,0,1,float16,float16,0,9.545802434285482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,64,128,1,float16,fp8,0,0.8490080038706461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,64,128,1,fp8,fp8,0,1.0891040166219075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,64,0,1,fp8,fp8,0,8.649845123291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,64,0,1,float16,fp8,0,9.522794723510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,64,128,1,float16,float16,0,0.7257920106252035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,64,0,1,float16,float16,0,5.14411195119222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,64,128,1,float16,fp8,0,0.721951961517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,64,0,1,float16,float16,0,9.635712305704752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,64,0,1,fp8,fp8,0,8.667269388834635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,64,0,1,float16,fp8,0,4.908447901407878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,64,128,1,fp8,fp8,0,0.9814879894256592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,64,0,1,float16,fp8,0,9.532314936319986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,64,0,1,fp8,fp8,0,4.495312054951985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,64,128,1,float16,float16,0,0.7613706588745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,64,128,1,float16,fp8,0,0.772325356801351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,64,128,1,fp8,fp8,0,0.9991520245869955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,64,0,1,float16,float16,0,5.036399841308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,64,128,1,float16,float16,0,0.7939306894938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,64,128,1,float16,fp8,0,0.7917439937591553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,64,128,1,fp8,fp8,0,1.0113653341929119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,64,0,1,fp8,fp8,0,4.357920010884603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,64,0,1,float16,fp8,0,5.031194686889648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,64,0,1,float16,float16,0,5.052341461181641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,64,128,1,float16,float16,0,0.8066026369730631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,64,0,1,fp8,fp8,0,4.3585866292317705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,64,0,1,float16,fp8,0,5.067328135172526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,64,128,1,float16,fp8,0,0.8150880336761475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,64,128,1,fp8,fp8,0,1.0292800267537434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,64,128,1,float16,float16,0,0.4363786776860555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,64,0,1,float16,float16,0,5.111509323120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,64,128,1,float16,fp8,0,0.43610668182373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,64,128,1,fp8,fp8,0,0.5860000054041544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,64,0,1,fp8,fp8,0,4.406309445699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,64,0,1,float16,fp8,0,4.795402526855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,64,0,1,float16,float16,0,5.079786618550618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,64,128,1,float16,float16,0,0.3999573389689128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,64,0,1,float16,float16,0,2.541754722595215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,64,128,1,float16,fp8,0,0.40866132577260333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,64,128,1,fp8,fp8,0,0.5179093281428019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,64,0,1,float16,fp8,0,4.984309196472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,64,0,1,fp8,fp8,0,4.438720067342122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,64,0,1,float16,fp8,0,2.5228959719340005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,64,0,1,fp8,fp8,0,2.303818702697754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,64,128,1,float16,float16,0,0.40354132652282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,64,0,1,float16,float16,0,2.461402734120687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,64,128,1,float16,fp8,0,0.39875201384226483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,64,128,1,fp8,fp8,0,0.5363093217213949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,64,0,1,float16,fp8,0,2.53929074605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,64,128,1,float16,float16,0,0.4075466791788737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,64,128,1,float16,fp8,0,0.41276800632476807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,64,0,1,fp8,fp8,0,2.244154612223307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,64,128,1,fp8,fp8,0,0.5485066572825114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,64,0,1,float16,float16,0,2.507808049519857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,64,128,1,float16,float16,0,0.4145333369572957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,64,0,1,float16,fp8,0,2.5611519813537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,64,0,1,fp8,fp8,0,2.2531092961629233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,64,128,1,float16,fp8,0,0.41571199893951416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,64,0,1,float16,float16,0,2.4634559949239097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,64,128,1,fp8,fp8,0,0.5447253386179606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,64,128,1,float16,float16,0,0.2889066735903422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,64,0,1,float16,fp8,0,2.452143987019857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,64,0,1,fp8,fp8,0,2.260261376698812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,64,128,1,float16,fp8,0,0.2888480027516683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,64,0,1,float16,float16,0,2.4570560455322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,64,128,1,fp8,fp8,0,0.3611840009689331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,64,0,1,float16,float16,0,1.3698453903198242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,64,0,1,float16,fp8,0,2.4622186024983725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,64,128,1,float16,float16,0,0.2873599926630656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,64,0,1,fp8,fp8,0,2.314746697743734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,64,0,1,float16,fp8,0,1.3730079332987468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,64,128,1,float16,fp8,0,0.2903839945793152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,64,0,1,fp8,fp8,0,1.1652053197224934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,64,128,1,fp8,fp8,0,0.3519253333409627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,64,0,1,float16,float16,0,1.3474453290303547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,64,128,1,float16,float16,0,0.2901866634686788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,64,128,1,float16,fp8,0,0.2879146734873454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,64,128,1,fp8,fp8,0,0.3518079916636149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,64,0,1,float16,fp8,0,1.3613707224527996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,64,0,1,fp8,fp8,0,1.1694933573404949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,64,0,1,float16,float16,0,1.3574934005737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,64,128,1,float16,float16,0,0.28799466292063397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,64,128,1,float16,fp8,0,0.2919253309567769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,64,128,1,fp8,fp8,0,0.35311468442281085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,64,0,1,fp8,fp8,0,1.1892053286234539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,64,0,1,float16,fp8,0,1.3395253817240398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,64,128,1,float16,float16,0,0.29022934039433795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,64,0,1,float16,float16,0,1.3506399790445964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,64,128,1,float16,fp8,0,0.2887093424797058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,64,128,1,fp8,fp8,0,0.35075199604034424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,64,0,1,fp8,fp8,0,1.1659039656321208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,64,0,1,float16,fp8,0,1.349397341410319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,64,0,1,float16,float16,0,1.348581314086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,64,0,1,float16,fp8,0,1.3503360748291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,64,0,1,fp8,fp8,0,1.1712693373362224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,64,128,1,float16,float16,0,1.0993706385294597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,64,128,1,float16,fp8,0,1.097061316172282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,64,128,1,fp8,fp8,0,1.4154666264851887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,64,128,1,float16,float16,0,1.1540160179138184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,64,128,1,float16,fp8,0,1.1549973487854004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,64,128,1,fp8,fp8,0,1.4639093081156414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,64,128,1,float16,float16,0,1.172426700592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,64,0,1,float16,float16,0,5.528373082478841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,64,0,1,float16,fp8,0,5.526261647542317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,64,0,1,fp8,fp8,0,5.036959966023763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,64,0,1,float16,float16,0,5.560378392537435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,64,128,1,float16,fp8,0,1.159120003382365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,64,128,1,fp8,fp8,0,1.4757226308186848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,64,0,1,fp8,fp8,0,5.0770721435546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,64,0,1,float16,fp8,0,5.770730972290039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,64,128,1,float16,float16,0,1.2054239908854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,64,128,1,float16,fp8,0,1.180997371673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,64,128,1,fp8,fp8,0,1.5143839518229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,64,128,1,float16,float16,0,0.6440106630325317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,64,0,1,float16,float16,0,5.784389495849609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,64,0,1,fp8,fp8,0,5.11028258005778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,64,128,1,float16,fp8,0,0.6264640092849731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,64,128,1,fp8,fp8,0,0.8365920384724935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,64,0,1,float16,fp8,0,5.693855921427409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,64,0,1,float16,float16,0,5.733066558837891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,64,128,1,float16,float16,0,0.5590293407440186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,64,0,1,float16,float16,0,2.891946792602539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,64,0,1,fp8,fp8,0,5.152112007141113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,64,128,1,float16,fp8,0,0.5447306632995605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,64,0,1,float16,fp8,0,5.829872131347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,64,128,1,fp8,fp8,0,0.7194933096567789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,64,0,1,fp8,fp8,0,2.6505813598632812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,64,0,1,float16,fp8,0,2.866373380025228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,64,128,1,float16,float16,0,0.5631999969482422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,64,128,1,float16,fp8,0,0.555242657661438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,64,0,1,float16,float16,0,2.82262388865153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,64,128,1,fp8,fp8,0,0.7308213710784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,64,128,1,float16,float16,0,0.5719786485036215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,64,0,1,float16,fp8,0,2.8641014099121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,64,128,1,float16,fp8,0,0.5572746594746908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,64,0,1,fp8,fp8,0,2.5237226486206055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,64,128,1,fp8,fp8,0,0.752021312713623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,64,0,1,float16,float16,0,2.819210688273112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,64,128,1,float16,float16,0,0.5862933397293091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,64,0,1,fp8,fp8,0,2.5758132934570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,64,0,1,float16,fp8,0,2.807077407836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,64,128,1,float16,fp8,0,0.5847520033518473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,64,0,1,float16,float16,0,2.824469248453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,64,128,1,fp8,fp8,0,0.7743039925893148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,64,128,1,float16,float16,0,0.3322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,64,0,1,fp8,fp8,0,2.5567520459493003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,64,128,1,float16,fp8,0,0.32866134246190387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,64,0,1,float16,fp8,0,2.857237180074056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,64,128,1,fp8,fp8,0,0.43457067012786865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,64,0,1,float16,float16,0,1.472373326619466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,64,0,1,float16,float16,0,2.9851147333780923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,64,128,1,float16,float16,0,0.314191997051239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,64,128,1,float16,fp8,0,0.3011626601219177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,64,0,1,fp8,fp8,0,2.650965372721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,64,0,1,float16,fp8,0,2.8184585571289062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,64,128,1,fp8,fp8,0,0.41012267271677655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,64,0,1,float16,fp8,0,1.458847999572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,64,0,1,fp8,fp8,0,1.3327893416086833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,64,0,1,float16,float16,0,1.4776320457458496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,64,128,1,float16,float16,0,0.3069546620051066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,64,128,1,float16,fp8,0,0.3107093373934428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,64,128,1,fp8,fp8,0,0.41065065066019696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,64,0,1,float16,fp8,0,1.4751946131388347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,64,128,1,float16,float16,0,0.31726400057474774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,64,0,1,fp8,fp8,0,1.2677813371022542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,64,0,1,float16,float16,0,1.5589386622111003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,64,128,1,float16,fp8,0,0.3166773319244385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,64,128,1,fp8,fp8,0,0.4132213195164998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,64,0,1,fp8,fp8,0,1.2782560189565022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,64,0,1,float16,fp8,0,1.4759519894917805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,64,0,1,float16,float16,0,1.4565332730611165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,64,128,1,float16,float16,0,0.3242186705271403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,64,128,1,float16,fp8,0,0.31644266843795776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,64,0,1,float16,fp8,0,1.4841440518697102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,64,128,1,fp8,fp8,0,0.4256480137507121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,64,0,1,fp8,fp8,0,1.301743984222412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,64,128,1,float16,float16,0,0.2222506602605184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,64,128,1,float16,fp8,0,0.2249173323313395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,64,0,1,float16,float16,0,1.4637707074483235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,64,128,1,fp8,fp8,0,0.2748639980951945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,64,0,1,float16,float16,0,0.7757546901702881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,64,0,1,float16,fp8,0,1.4600639343261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,64,0,1,fp8,fp8,0,1.3183200359344482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,64,128,1,float16,float16,0,0.221178670724233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,64,0,1,float16,fp8,0,0.7630826632181803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,64,0,1,fp8,fp8,0,0.7170720100402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,64,128,1,float16,fp8,0,0.22581332921981812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,64,128,1,fp8,fp8,0,0.2737920085589091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,64,0,1,float16,float16,0,0.7633866469065348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,64,128,1,float16,float16,0,0.22451200087865195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,64,0,1,fp8,fp8,0,0.7154186566670736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,64,0,1,float16,fp8,0,0.7796746889750162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,64,128,1,float16,fp8,0,0.22483734289805093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,64,128,1,fp8,fp8,0,0.273525337378184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,64,0,1,float16,float16,0,0.7772426605224609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,64,128,1,float16,float16,0,0.22485866149266562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,64,0,1,float16,fp8,0,0.7643253008524576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,64,0,1,fp8,fp8,0,0.7176213264465332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,64,128,1,float16,fp8,0,0.22620266675949097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,64,128,1,fp8,fp8,0,0.2760159969329834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,64,128,1,float16,float16,0,0.22494399547576904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,64,0,1,float16,float16,0,0.7672800223032633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,64,128,1,float16,fp8,0,0.22793066501617432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,64,0,1,fp8,fp8,0,0.7170666853586832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,64,0,1,float16,fp8,0,0.7803733348846436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,64,128,1,fp8,fp8,0,0.2762613296508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,64,0,1,float16,float16,0,0.7646613121032715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,64,0,1,float16,fp8,0,0.7789333661397299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,64,0,1,fp8,fp8,0,0.7076266606648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,64,128,1,float16,float16,0,0.9151253700256348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,64,128,1,float16,fp8,0,0.9025493462880453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,64,128,1,fp8,fp8,0,1.183557351430257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,64,128,1,float16,float16,0,0.9733440081278483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,64,128,1,float16,fp8,0,0.9551680088043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,64,128,1,fp8,fp8,0,1.2359946568806965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,64,128,1,float16,float16,0,0.977125326792399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,64,0,1,float16,float16,0,3.998288154602051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,64,0,1,fp8,fp8,0,3.642762819925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,64,0,1,float16,fp8,0,4.12338129679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,64,0,1,float16,float16,0,4.128896077473958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,64,128,1,float16,fp8,0,0.966591993967692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,64,128,1,fp8,fp8,0,1.2547893524169922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,64,0,1,fp8,fp8,0,3.6558825174967446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,64,0,1,float16,fp8,0,4.027685483296712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,64,128,1,float16,float16,0,1.0036160151163738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,64,128,1,float16,fp8,0,0.9901813666025797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,64,0,1,float16,float16,0,4.105696042378743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,64,128,1,fp8,fp8,0,1.269221305847168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,64,128,1,float16,float16,0,0.5340533256530762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,64,128,1,float16,fp8,0,0.5193813244501749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,64,0,1,fp8,fp8,0,3.667146682739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,64,0,1,float16,fp8,0,4.161733309427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,64,128,1,fp8,fp8,0,0.6842666467030843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,64,0,1,float16,float16,0,2.056826591491699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,64,128,1,float16,float16,0,0.47674131393432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,64,0,1,float16,float16,0,4.096000035603841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,64,0,1,fp8,fp8,0,3.6943146387736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,64,0,1,float16,fp8,0,4.180287996927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,64,128,1,float16,fp8,0,0.4548480113347371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,64,0,1,fp8,fp8,0,1.9077226320902507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,64,0,1,float16,fp8,0,2.0734826723734536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,64,128,1,fp8,fp8,0,0.6198506752649943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,64,0,1,float16,float16,0,2.102479934692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,64,128,1,float16,float16,0,0.476911981900533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,64,128,1,float16,fp8,0,0.4582560062408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,64,128,1,fp8,fp8,0,0.6283146540323893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,64,0,1,float16,fp8,0,1.9879627227783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,64,128,1,float16,float16,0,0.4753919839859009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,64,0,1,fp8,fp8,0,1.8265280723571777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,64,128,1,float16,fp8,0,0.47785067558288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,64,0,1,float16,float16,0,2.017845312754313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,64,128,1,fp8,fp8,0,0.6308266719182333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,64,128,1,float16,float16,0,0.48957868417104083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,64,0,1,fp8,fp8,0,1.8795679410298665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,64,0,1,float16,fp8,0,1.991205374399821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,64,0,1,float16,float16,0,2.090826670328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,64,128,1,float16,fp8,0,0.4836266835530599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,64,128,1,float16,float16,0,0.2878933350245158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,64,128,1,fp8,fp8,0,0.6485333442687988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,64,0,1,float16,fp8,0,1.9977812767028809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,64,0,1,fp8,fp8,0,1.8499040603637695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,64,128,1,float16,fp8,0,0.28198399146397907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,64,0,1,float16,float16,0,2.0237013498942056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,64,128,1,fp8,fp8,0,0.37789865334828693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,64,0,1,float16,float16,0,1.0809439818064372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,64,0,1,fp8,fp8,0,1.8813865979512532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,64,0,1,float16,fp8,0,2.010138670603434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,64,128,1,float16,float16,0,0.27090134223302204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,64,0,1,float16,fp8,0,1.0642133553822835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,64,128,1,float16,fp8,0,0.26043200492858887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,64,0,1,fp8,fp8,0,0.9458933671315511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,64,128,1,fp8,fp8,0,0.33670934041341144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,64,0,1,float16,float16,0,1.0691466331481934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,64,128,1,float16,float16,0,0.268613338470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,64,128,1,float16,fp8,0,0.2583893338839213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,64,128,1,fp8,fp8,0,0.3394346634546916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,64,0,1,fp8,fp8,0,0.9588426748911539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,64,0,1,float16,fp8,0,1.0460533301035564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,64,128,1,float16,float16,0,0.263973335425059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,64,0,1,float16,float16,0,1.0617226759592693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,64,128,1,float16,fp8,0,0.2639039953549703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,64,0,1,float16,fp8,0,1.0673279762268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,64,128,1,fp8,fp8,0,0.35146133104960126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,64,0,1,fp8,fp8,0,0.9475306669871012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,64,128,1,float16,float16,0,0.27001599470774335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,64,0,1,float16,float16,0,1.054154634475708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,64,128,1,float16,fp8,0,0.2738719979921977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,64,0,1,float16,fp8,0,1.051477352778117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,64,0,1,fp8,fp8,0,0.9368213017781576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,64,128,1,fp8,fp8,0,0.36219199498494464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,64,128,1,float16,float16,0,0.1911733349164327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,64,0,1,float16,float16,0,1.077957312266032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,64,128,1,float16,fp8,0,0.19218132893244425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,64,128,1,fp8,fp8,0,0.23626667261123657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,64,0,1,float16,float16,0,0.5571839809417725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,64,0,1,fp8,fp8,0,0.9562453428904215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,64,0,1,float16,fp8,0,1.0604480107625325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,64,128,1,float16,float16,0,0.18997865915298462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,64,0,1,float16,fp8,0,0.5667946736017863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,64,128,1,float16,fp8,0,0.19025067488352457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,64,0,1,fp8,fp8,0,0.525493343671163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,64,128,1,fp8,fp8,0,0.2360693415006002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,64,0,1,float16,float16,0,0.5579466819763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,64,128,1,float16,float16,0,0.1904053290685018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,64,0,1,float16,fp8,0,0.557749350865682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,64,0,1,fp8,fp8,0,0.5245973269144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,64,128,1,float16,fp8,0,0.18997333447138467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,64,128,1,fp8,fp8,0,0.23667732874552408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,64,0,1,float16,float16,0,0.5617813269297282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,64,128,1,float16,float16,0,0.1906720002492269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,64,128,1,float16,fp8,0,0.19082132975260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,64,0,1,float16,fp8,0,0.5590933163960775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,64,0,1,fp8,fp8,0,0.5255946715672811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,64,128,1,fp8,fp8,0,0.23663467168807983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,64,0,1,float16,float16,0,0.557151993115743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,64,0,1,fp8,fp8,0,0.5254613161087036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,64,128,1,float16,float16,0,0.1928159991900126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,64,0,1,float16,fp8,0,0.5623146692911783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,64,128,1,float16,fp8,0,0.19138665994008383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,64,0,1,float16,float16,0,0.5593599875768026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,64,128,1,fp8,fp8,0,0.23690134286880493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,64,0,1,float16,fp8,0,0.5585813522338867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,64,0,1,fp8,fp8,0,0.5259519815444946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,64,128,1,float16,float16,0,1.4489226341247559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,64,128,1,float16,fp8,0,1.4510666529337566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,64,128,1,fp8,fp8,0,1.8767414093017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,64,128,1,float16,float16,0,1.5216107368469238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,64,128,1,float16,fp8,0,1.506165345509847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,64,128,1,fp8,fp8,0,1.9416054089864094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,64,0,1,float16,float16,0,5.357002894083659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,64,0,1,fp8,fp8,0,4.76254399617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,64,0,1,float16,fp8,0,5.240410804748535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,64,128,1,float16,float16,0,1.5401546160380046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,64,0,1,float16,float16,0,5.536938349405925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,64,128,1,float16,fp8,0,1.5416852633158367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,64,0,1,fp8,fp8,0,4.815322558085124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,64,128,1,fp8,fp8,0,1.9409653345743816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,64,0,1,float16,fp8,0,5.476981480916341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,64,128,1,float16,float16,0,1.6125706036885579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,64,128,1,float16,fp8,0,1.5631252924601238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,64,0,1,float16,float16,0,5.3035891850789385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,64,128,1,fp8,fp8,0,1.9914026260375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,64,128,1,float16,float16,0,0.8485013643900553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,64,0,1,fp8,fp8,0,4.83789857228597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,64,128,1,float16,fp8,0,0.836026668548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,64,0,1,float16,fp8,0,5.39137585957845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,64,128,1,fp8,fp8,0,1.0698613325754802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,64,128,1,float16,float16,0,0.711189349492391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,64,0,1,float16,float16,0,2.8183412551879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,64,0,1,float16,float16,0,5.313146591186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,64,128,1,float16,fp8,0,0.7002293268839518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,64,0,1,float16,fp8,0,5.520383834838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,64,0,1,float16,fp8,0,2.8076906204223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,64,128,1,fp8,fp8,0,0.9605279763539633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,64,0,1,fp8,fp8,0,2.5295146306355796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,64,0,1,fp8,fp8,0,4.912416140238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,64,128,1,float16,float16,0,0.774394671122233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,64,0,1,float16,float16,0,2.597386678059896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,64,128,1,float16,fp8,0,0.7617493470509847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,64,128,1,fp8,fp8,0,0.9670186837514242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,64,0,1,float16,fp8,0,2.607205390930176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,64,128,1,float16,float16,0,0.7699200312296549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,64,0,1,fp8,fp8,0,2.4068800608317056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,64,0,1,float16,float16,0,2.6670398712158203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,64,128,1,float16,fp8,0,0.7620000044504801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,64,0,1,float16,fp8,0,2.7875146865844727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,64,128,1,fp8,fp8,0,0.9945226510365804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,64,0,1,fp8,fp8,0,2.4979093869527182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,64,128,1,float16,float16,0,0.7946613629659017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,64,0,1,float16,float16,0,2.696330706278483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,64,128,1,float16,fp8,0,0.7877546946207682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,64,128,1,fp8,fp8,0,1.0393226941426594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,64,0,1,float16,fp8,0,2.750768025716146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,64,128,1,float16,float16,0,0.4225706656773885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,64,0,1,fp8,fp8,0,2.463040033976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,64,128,1,float16,fp8,0,0.4256800015767415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,64,0,1,float16,float16,0,2.7177600860595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,64,128,1,fp8,fp8,0,0.5461653470993042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,64,0,1,float16,float16,0,1.3805707295735676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,64,128,1,float16,float16,0,0.37273065249125165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,64,0,1,float16,fp8,0,2.71503480275472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,64,0,1,fp8,fp8,0,2.517162640889486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,64,0,1,float16,fp8,0,1.3723573684692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,64,128,1,float16,fp8,0,0.3684533437093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,64,0,1,fp8,fp8,0,1.3058826923370361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,64,128,1,fp8,fp8,0,0.49029334386189777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,64,0,1,float16,float16,0,1.337824026743571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,64,128,1,float16,float16,0,0.37697601318359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,64,128,1,float16,fp8,0,0.37328533331553143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,64,128,1,fp8,fp8,0,0.4977333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,64,0,1,fp8,fp8,0,1.2347359657287598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,64,0,1,float16,fp8,0,1.3556532859802246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,64,0,1,float16,float16,0,1.341477394104004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,64,128,1,float16,float16,0,0.3835200071334839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,64,128,1,float16,fp8,0,0.37833599249521893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,64,0,1,float16,fp8,0,1.3565707206726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,64,128,1,fp8,fp8,0,0.5133493343989054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,64,0,1,fp8,fp8,0,1.2497333685557048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,64,0,1,float16,float16,0,1.3720426559448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,64,128,1,float16,float16,0,0.39262934525807697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,64,128,1,float16,fp8,0,0.3961493174235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,64,128,1,fp8,fp8,0,0.5271679957707723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,64,128,1,float16,float16,0,0.22847465674082437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,64,0,1,float16,fp8,0,1.3708693186442058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,64,0,1,fp8,fp8,0,1.2500800291697185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,64,0,1,float16,float16,0,1.3776586850484211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,64,128,1,float16,fp8,0,0.23688532908757529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,64,128,1,fp8,fp8,0,0.3004693388938904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,64,0,1,float16,float16,0,0.7480639616648356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,64,0,1,fp8,fp8,0,1.2909013430277507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,64,0,1,float16,fp8,0,1.3654133478800456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,64,128,1,float16,float16,0,0.21563732624053955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,64,0,1,fp8,fp8,0,0.6416000127792358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,64,128,1,float16,fp8,0,0.20935465892155966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,64,0,1,float16,fp8,0,0.7467466990152994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,64,128,1,fp8,fp8,0,0.28514132897059125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,64,0,1,float16,float16,0,0.7254719734191895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,64,128,1,float16,float16,0,0.21694932381312051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,64,0,1,float16,fp8,0,0.7230346997578939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,64,128,1,float16,fp8,0,0.21118400494257608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,64,0,1,fp8,fp8,0,0.6265013217926025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,64,128,1,fp8,fp8,0,0.2871466676394145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,64,0,1,float16,float16,0,0.7135787010192871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,64,128,1,float16,float16,0,0.21593066056569418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,64,0,1,float16,fp8,0,0.7247412999471029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,64,128,1,float16,fp8,0,0.21436800559361777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,64,0,1,fp8,fp8,0,0.635696013768514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,64,128,1,fp8,fp8,0,0.2871359984079997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,64,0,1,float16,float16,0,0.7301119963328043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,64,128,1,float16,float16,0,0.21960532665252686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,64,0,1,fp8,fp8,0,0.6259573300679525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,64,128,1,float16,fp8,0,0.21972266832987467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,64,128,1,fp8,fp8,0,0.2957599957784017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,64,0,1,float16,fp8,0,0.7297493616739908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,64,0,1,float16,float16,0,0.7355946699778239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,64,128,1,float16,float16,0,0.15685866276423135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,64,128,1,float16,fp8,0,0.15940266847610474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,64,0,1,float16,float16,0,0.3917173147201538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,64,0,1,fp8,fp8,0,0.6437439918518066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,64,128,1,fp8,fp8,0,0.19875733057657877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,64,0,1,float16,fp8,0,0.7203786373138428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,64,128,1,float16,float16,0,0.15527466932932535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,64,0,1,float16,fp8,0,0.3906346559524536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,64,0,1,fp8,fp8,0,0.3652213414510091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,64,128,1,float16,fp8,0,0.15794133146603903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,64,128,1,fp8,fp8,0,0.19636799891789755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,64,0,1,float16,float16,0,0.38994133472442627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,64,128,1,float16,float16,0,0.15518933534622192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,64,0,1,float16,fp8,0,0.39107731978098553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,64,0,1,fp8,fp8,0,0.3590986728668213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,64,128,1,float16,fp8,0,0.15707733233769736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,64,0,1,float16,float16,0,0.3914560079574585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,64,0,1,float16,fp8,0,0.3893653154373169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,64,128,1,fp8,fp8,0,0.19537067413330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,64,128,1,float16,float16,0,0.1567200024922689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,64,0,1,fp8,fp8,0,0.3635893265406291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,64,128,1,float16,fp8,0,0.156058669090271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,64,128,1,fp8,fp8,0,0.19709332784016928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,64,0,1,float16,float16,0,0.3904586633046468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,64,128,1,float16,float16,0,0.15709333618481955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,64,0,1,float16,fp8,0,0.3919893503189087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,64,0,1,fp8,fp8,0,0.36648531754811603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,64,128,1,float16,fp8,0,0.1581013302008311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,64,0,1,float16,float16,0,0.3846293290456136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,64,128,1,fp8,fp8,0,0.19716266791025797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,64,0,1,float16,fp8,0,0.39055466651916504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,64,0,1,fp8,fp8,0,0.36191999912261963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,64,128,1,float16,float16,0,1.0992159843444824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,64,128,1,float16,fp8,0,1.0723146597544353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,64,128,1,fp8,fp8,0,1.3923786481221516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,64,128,1,float16,float16,0,1.1421866416931152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,64,128,1,float16,fp8,0,1.1338133017222087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,64,0,1,float16,float16,0,3.101557413736979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,64,0,1,fp8,fp8,0,2.9001547495524087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,64,0,1,float16,fp8,0,3.0934505462646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,64,128,1,fp8,fp8,0,1.4448213577270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,64,0,1,float16,float16,0,3.2369066874186196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,64,128,1,float16,float16,0,1.159157355626424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,64,128,1,float16,fp8,0,1.1440746784210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,64,128,1,fp8,fp8,0,1.4596853256225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,64,0,1,float16,fp8,0,3.1686986287434897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,64,0,1,fp8,fp8,0,2.921450614929199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,64,128,1,float16,float16,0,1.1917333602905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,64,0,1,float16,float16,0,3.2411254247029624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,64,128,1,float16,fp8,0,1.175594647725423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,64,128,1,fp8,fp8,0,1.4927253723144531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,64,128,1,float16,float16,0,0.6334880193074545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,64,0,1,float16,fp8,0,3.166207949320475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,64,0,1,fp8,fp8,0,2.9405012130737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,64,128,1,float16,fp8,0,0.6178239981333414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,64,0,1,float16,float16,0,3.3562399546305337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,64,128,1,fp8,fp8,0,0.812175989151001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,64,0,1,float16,float16,0,1.6604746182759602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,64,128,1,float16,float16,0,0.5260639985402426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,64,0,1,float16,fp8,0,3.26908270517985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,64,128,1,float16,fp8,0,0.5232479969660441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,64,0,1,fp8,fp8,0,1.5534292856852214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,64,0,1,float16,fp8,0,1.6783413887023926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,64,0,1,fp8,fp8,0,2.963141441345215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,64,128,1,fp8,fp8,0,0.7028799851735433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,64,0,1,float16,float16,0,1.6057440439860027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,64,128,1,float16,float16,0,0.5448533296585083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,64,128,1,float16,fp8,0,0.5471359888712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,64,0,1,float16,fp8,0,1.5645866394042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,64,128,1,fp8,fp8,0,0.7190133730570475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,64,0,1,fp8,fp8,0,1.4456639289855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,64,128,1,float16,float16,0,0.5544373194376627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,64,0,1,float16,float16,0,1.563205401102702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,64,128,1,float16,fp8,0,0.5530080000559489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,64,0,1,float16,fp8,0,1.5780000686645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,64,0,1,fp8,fp8,0,1.4621334075927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,64,128,1,fp8,fp8,0,0.7349759737650553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,64,128,1,float16,float16,0,0.593936006228129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,64,0,1,float16,float16,0,1.5825014114379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,64,128,1,float16,fp8,0,0.5742133458455404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,64,0,1,fp8,fp8,0,1.4786826769510906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,64,128,1,fp8,fp8,0,0.765226682027181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,64,0,1,float16,fp8,0,1.5685973167419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,64,128,1,float16,float16,0,0.3255839943885803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,64,0,1,float16,float16,0,1.6110506057739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,64,128,1,float16,fp8,0,0.31590932607650757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,64,128,1,fp8,fp8,0,0.4124693473180135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,64,0,1,float16,float16,0,0.8521813551584879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,64,0,1,float16,fp8,0,1.6215146382649739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,64,128,1,float16,float16,0,0.28965334097544354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,64,0,1,fp8,fp8,0,0.7670293649037679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,64,0,1,float16,fp8,0,0.8552853266398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,64,128,1,float16,fp8,0,0.28114134073257446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,64,0,1,fp8,fp8,0,1.5333600044250488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,64,128,1,fp8,fp8,0,0.38150401910146076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,64,0,1,float16,float16,0,0.8287999629974365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,64,128,1,float16,float16,0,0.2900320092837016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,64,128,1,float16,fp8,0,0.2842079997062683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,64,0,1,fp8,fp8,0,0.7335253556569418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,64,0,1,float16,fp8,0,0.8242773214975992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,64,128,1,fp8,fp8,0,0.3872906764348348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,64,128,1,float16,float16,0,0.2935306628545125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,64,0,1,float16,float16,0,0.829749345779419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,64,128,1,float16,fp8,0,0.29420266548792523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,64,0,1,fp8,fp8,0,0.7407573064168295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,64,0,1,float16,fp8,0,0.8204906781514486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,64,128,1,fp8,fp8,0,0.39258134365081787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,64,0,1,float16,float16,0,0.8185653686523438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,64,128,1,float16,float16,0,0.30461867650349933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,64,128,1,float16,fp8,0,0.30499200026194256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,64,0,1,float16,fp8,0,0.8293120066324869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,64,0,1,fp8,fp8,0,0.7304853598276774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,64,128,1,fp8,fp8,0,0.40141332149505615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,64,128,1,float16,float16,0,0.18119466304779053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,64,0,1,float16,float16,0,0.839850664138794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,64,128,1,float16,fp8,0,0.18196799357732138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,64,0,1,float16,float16,0,0.4227679967880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,64,128,1,fp8,fp8,0,0.2357920010884603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,64,0,1,float16,fp8,0,0.8414506912231445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,64,0,1,fp8,fp8,0,0.7386399904886881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,64,128,1,float16,float16,0,0.16563733418782553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,64,0,1,float16,fp8,0,0.42958935101826984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,64,0,1,fp8,fp8,0,0.4057226578394572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,64,128,1,float16,fp8,0,0.16242133577664694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,64,0,1,float16,float16,0,0.4168906609217326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,64,128,1,fp8,fp8,0,0.21767999728520712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,64,128,1,float16,float16,0,0.16782933473587036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,64,0,1,float16,fp8,0,0.4161706765492757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,64,0,1,fp8,fp8,0,0.3893119891484578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,64,128,1,float16,fp8,0,0.16320533553759256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,64,0,1,float16,float16,0,0.41091732184092206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,64,128,1,fp8,fp8,0,0.22258667151133218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,64,0,1,float16,fp8,0,0.40904001394907635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,64,128,1,float16,float16,0,0.169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,64,0,1,fp8,fp8,0,0.388810674349467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,64,128,1,float16,fp8,0,0.16607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,64,0,1,float16,float16,0,0.418559988339742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,64,128,1,fp8,fp8,0,0.2225066622098287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,64,128,1,float16,float16,0,0.17305066188176474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,64,0,1,float16,fp8,0,0.41180264949798584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,64,0,1,fp8,fp8,0,0.3861813147862752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,64,128,1,float16,fp8,0,0.17052799463272095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,64,0,1,float16,float16,0,0.4203893343607585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,64,128,1,fp8,fp8,0,0.23052799701690674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,64,128,1,float16,float16,0,0.12494933605194092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,64,0,1,float16,fp8,0,0.4152373472849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,64,0,1,fp8,fp8,0,0.3956906795501709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,64,128,1,float16,fp8,0,0.12428800264994304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,64,0,1,float16,float16,0,0.2446026603380839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,64,128,1,fp8,fp8,0,0.15684266885121664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,64,0,1,float16,fp8,0,0.24676799774169922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,64,0,1,fp8,fp8,0,0.23024000724156699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,64,128,1,float16,float16,0,0.12083199620246887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,64,128,1,float16,fp8,0,0.12086932857831319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,64,0,1,float16,float16,0,0.24547733863194784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,64,128,1,fp8,fp8,0,0.14689600467681885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,64,0,1,float16,fp8,0,0.24395734071731567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,64,128,1,float16,float16,0,0.12139200170834859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,64,0,1,fp8,fp8,0,0.22952000300089517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,64,0,1,float16,float16,0,0.24331732590993246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,64,128,1,float16,fp8,0,0.1211946705977122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,64,128,1,float16,float16,0,0.12203733126322429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,64,128,1,fp8,fp8,0,0.1529706617196401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,64,0,1,float16,fp8,0,0.24202666680018106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,64,0,1,fp8,fp8,0,0.22900799910227457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,64,0,1,float16,float16,0,0.24562132358551025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,64,128,1,float16,fp8,0,0.1234879990418752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,64,128,1,fp8,fp8,0,0.15427199999491373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,64,0,1,float16,fp8,0,0.24330133199691772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,64,0,1,fp8,fp8,0,0.22946133216222128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,64,128,1,float16,float16,0,0.12338667114575703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,64,0,1,float16,float16,0,0.24517333507537842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,64,128,1,float16,fp8,0,0.12401599685351054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,64,128,1,fp8,fp8,0,0.15576000014940897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,64,0,1,float16,fp8,0,0.24531733989715576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,64,0,1,fp8,fp8,0,0.22992533445358276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,64,128,1,float16,float16,0,1.441424051920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,64,128,1,float16,fp8,0,1.4357333183288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,64,128,1,fp8,fp8,0,1.8366773923238118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,64,128,1,float16,float16,0,1.5072800318400066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,64,0,1,float16,float16,0,3.094010670979818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,64,0,1,fp8,fp8,0,2.8575572967529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,64,0,1,float16,fp8,0,3.0823841094970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,64,128,1,float16,fp8,0,1.4981759389241536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,64,128,1,fp8,fp8,0,1.9069280624389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,64,0,1,float16,float16,0,3.1852798461914062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,64,128,1,float16,float16,0,1.5329599380493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,64,128,1,float16,fp8,0,1.5115680694580078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,64,128,1,fp8,fp8,0,1.9305119514465332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,64,0,1,float16,fp8,0,3.1212692260742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,64,0,1,fp8,fp8,0,2.9196106592814126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,64,0,1,float16,float16,0,3.177061398824056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,64,128,1,float16,float16,0,1.5857547124226887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,64,128,1,float16,fp8,0,1.5570720036824544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,64,0,1,float16,fp8,0,3.1450506846110025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,64,0,1,fp8,fp8,0,2.9395573933919272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,64,128,1,fp8,fp8,0,1.9797706604003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,64,128,1,float16,float16,0,0.8488372961680094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,64,0,1,float16,float16,0,3.260037422180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,64,128,1,float16,fp8,0,0.8260746796925863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,64,0,1,float16,fp8,0,3.191727956136068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,64,128,1,fp8,fp8,0,1.0603946844736736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,64,0,1,float16,float16,0,1.6673706372578938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,64,128,1,float16,float16,0,0.7102293173472086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,64,0,1,fp8,fp8,0,2.9917386372884116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,64,0,1,float16,fp8,0,1.6748533248901367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,64,0,1,fp8,fp8,0,1.5722452799479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,64,128,1,float16,fp8,0,0.6891679763793945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,64,128,1,fp8,fp8,0,0.915717363357544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,64,0,1,float16,float16,0,1.5247839291890461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,64,128,1,float16,float16,0,0.7396480242411295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,64,128,1,float16,fp8,0,0.7400213082631429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,64,0,1,float16,fp8,0,1.5278879801432292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,64,0,1,fp8,fp8,0,1.4289065996805828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,64,128,1,fp8,fp8,0,0.9658719698588053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,64,0,1,float16,float16,0,1.5948586463928223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,64,128,1,float16,float16,0,0.7551999886830648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,64,0,1,float16,fp8,0,1.58842134475708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,64,128,1,float16,fp8,0,0.759061336517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,64,0,1,fp8,fp8,0,1.4741013844807942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,64,128,1,fp8,fp8,0,0.9822879632314047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,64,0,1,float16,float16,0,1.586176077524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,64,128,1,float16,float16,0,0.7832213242848715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,64,0,1,float16,fp8,0,1.5844640731811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,64,128,1,float16,fp8,0,0.7819626331329346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,64,0,1,fp8,fp8,0,1.5148213704427083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,64,128,1,fp8,fp8,0,1.007141351699829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,64,0,1,float16,float16,0,1.6405760447184246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,64,128,1,float16,float16,0,0.43829333782196045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,64,0,1,float16,fp8,0,1.6243893305460613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,64,128,1,float16,fp8,0,0.4143893321355184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,64,0,1,float16,float16,0,0.8372106552124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,64,128,1,fp8,fp8,0,0.5410826603571574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,64,0,1,fp8,fp8,0,1.5046133995056152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,64,128,1,float16,float16,0,0.3667093515396118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,64,0,1,float16,fp8,0,0.8373546600341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,64,0,1,fp8,fp8,0,0.8075253168741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,64,128,1,float16,fp8,0,0.3551946481068929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,64,128,1,fp8,fp8,0,0.48527999718983966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,64,0,1,float16,float16,0,0.8033653100331625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,64,0,1,float16,fp8,0,0.7696159680684408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,64,128,1,float16,float16,0,0.37349867820739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,64,0,1,fp8,fp8,0,0.7419573465983073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,64,128,1,float16,fp8,0,0.3615146478017171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,64,0,1,float16,float16,0,0.7968746821085612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,64,128,1,fp8,fp8,0,0.490666667620341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,64,128,1,float16,float16,0,0.37697064876556396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,64,0,1,fp8,fp8,0,0.7429440021514893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,64,0,1,float16,fp8,0,0.7810400327046713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,64,128,1,float16,fp8,0,0.37123199303944904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,64,128,1,fp8,fp8,0,0.49346665541330975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,64,0,1,float16,float16,0,0.8065866629282633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,64,128,1,float16,float16,0,0.38601064682006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,64,0,1,float16,fp8,0,0.8025920391082764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,64,0,1,fp8,fp8,0,0.7451146443684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,64,128,1,float16,fp8,0,0.3834346532821655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,64,128,1,fp8,fp8,0,0.5094773372014364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,64,0,1,float16,float16,0,0.8019146919250488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,64,128,1,float16,float16,0,0.2237280011177063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,64,0,1,float16,fp8,0,0.7996640205383301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,64,128,1,float16,fp8,0,0.222762664159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,64,0,1,float16,float16,0,0.4386399984359741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,64,0,1,fp8,fp8,0,0.7646346886952718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,64,128,1,fp8,fp8,0,0.2913653254508972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,64,0,1,float16,fp8,0,0.4416533311208089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,64,128,1,float16,float16,0,0.19701866308848062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,64,0,1,fp8,fp8,0,0.40116266409556073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,64,128,1,float16,fp8,0,0.19538666804631552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,64,0,1,float16,float16,0,0.42291200160980225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,64,128,1,float16,float16,0,0.19934932390848795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,64,0,1,float16,fp8,0,0.41812801361083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,64,128,1,fp8,fp8,0,0.2654186685880025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,64,0,1,fp8,fp8,0,0.3747359911600749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,64,128,1,float16,fp8,0,0.19777067502339682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,64,0,1,float16,float16,0,0.425930658976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,64,128,1,fp8,fp8,0,0.2707573374112447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,64,0,1,float16,fp8,0,0.41544000307718915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,64,0,1,fp8,fp8,0,0.3768746852874756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,64,128,1,float16,float16,0,0.20444266001383463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,64,128,1,float16,fp8,0,0.20380266507466635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,64,128,1,fp8,fp8,0,0.27190933624903363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,64,0,1,float16,float16,0,0.42744000752766925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,64,0,1,float16,fp8,0,0.4256746768951416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,64,0,1,fp8,fp8,0,0.3803199927012126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,64,128,1,float16,float16,0,0.20930665731430054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,64,128,1,float16,fp8,0,0.20938666661580405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,64,0,1,float16,float16,0,0.4244906504948934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,64,128,1,fp8,fp8,0,0.27534933884938556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,64,128,1,float16,float16,0,0.128330667813619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,64,0,1,float16,fp8,0,0.42867199579874676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,64,128,1,fp8,fp8,0,0.1694773236910502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,64,0,1,fp8,fp8,0,0.38708265622456867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,64,0,1,float16,float16,0,0.22322134176890054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,64,128,1,float16,fp8,0,0.12966400384902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,64,0,1,float16,fp8,0,0.22502400477727255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,64,0,1,fp8,fp8,0,0.21669334173202515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,64,128,1,float16,float16,0,0.11812266707420349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,64,0,1,float16,float16,0,0.21373866001764932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,64,128,1,float16,fp8,0,0.11727999647458394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,64,128,1,fp8,fp8,0,0.15628266334533691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,64,0,1,float16,fp8,0,0.2137706677118937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,64,0,1,fp8,fp8,0,0.20482667287190756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,64,128,1,float16,float16,0,0.11786133050918579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,64,0,1,float16,float16,0,0.21679999430974325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,64,128,1,float16,fp8,0,0.11802132924397786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,64,128,1,fp8,fp8,0,0.15684266885121664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,64,0,1,float16,fp8,0,0.21450134118398032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,64,0,1,fp8,fp8,0,0.20397865772247314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,64,128,1,float16,float16,0,0.11840533216794331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,64,0,1,float16,float16,0,0.21785066525141397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,64,128,1,float16,fp8,0,0.11900800466537476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,64,128,1,fp8,fp8,0,0.16061866283416748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,64,0,1,float16,fp8,0,0.21859200795491537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,64,0,1,fp8,fp8,0,0.20786666870117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,64,128,1,float16,float16,0,0.12168000141779582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,64,0,1,float16,fp8,0,0.21978666385014853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,64,0,1,float16,float16,0,0.21915199359258017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,64,128,1,float16,fp8,0,0.12134400010108948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,64,128,1,fp8,fp8,0,0.16470932960510254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,64,0,1,fp8,fp8,0,0.20962133010228476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,64,128,1,float16,float16,0,0.09126933415730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,64,0,1,float16,float16,0,0.13807466626167297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,64,128,1,float16,fp8,0,0.09142399827639262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,64,128,1,fp8,fp8,0,0.11554666360219319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,64,0,1,float16,fp8,0,0.13741333285967508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,64,0,1,fp8,fp8,0,0.13035733501116434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,64,128,1,float16,float16,0,0.08691733082135518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,64,0,1,float16,float16,0,0.13529066244761148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,64,128,1,float16,fp8,0,0.0867786705493927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,64,128,1,fp8,fp8,0,0.10389866431554158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,64,0,1,float16,fp8,0,0.13475199540456137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,64,0,1,fp8,fp8,0,0.12773866454760233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,64,128,1,float16,float16,0,0.08779199918111165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,64,0,1,float16,float16,0,0.13527466853459677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,64,128,1,float16,fp8,0,0.08762133121490479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,64,128,1,fp8,fp8,0,0.10390399893124898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,64,0,1,float16,fp8,0,0.13496533036231995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,64,0,1,fp8,fp8,0,0.1274773379166921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,64,128,1,float16,float16,0,0.0881760021050771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,64,0,1,float16,float16,0,0.13525866468747458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,64,128,1,float16,fp8,0,0.08779733379681905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,64,128,1,fp8,fp8,0,0.10683733224868774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,64,0,1,float16,fp8,0,0.1345919966697693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,64,0,1,fp8,fp8,0,0.12777066230773926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,64,128,1,float16,float16,0,0.08989333113034566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,64,0,1,float16,float16,0,0.13582400480906168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,64,128,1,float16,fp8,0,0.08915733297665913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,64,128,1,fp8,fp8,0,0.1088853379090627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,64,0,1,float16,fp8,0,0.1359946628411611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,64,0,1,fp8,fp8,0,0.1286240021387736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,64,128,1,float16,float16,0,1.0590666929880779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,64,128,1,float16,fp8,0,1.047594706217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,64,128,1,fp8,fp8,0,1.3645866711934407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,64,0,1,float16,float16,0,1.89629332224528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,64,128,1,float16,float16,0,1.0935839811960857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,64,0,1,float16,fp8,0,1.8875786463419597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,64,0,1,fp8,fp8,0,1.7639573415120442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,64,128,1,float16,fp8,0,1.0852373441060383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,64,0,1,float16,float16,0,1.9385600090026855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,64,128,1,fp8,fp8,0,1.3891253471374512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,64,128,1,float16,float16,0,1.1198026339213054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,64,0,1,float16,fp8,0,1.9236639340718586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,64,0,1,fp8,fp8,0,1.7962026596069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,64,128,1,float16,fp8,0,1.1084266503651936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,64,0,1,float16,float16,0,1.95634126663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,64,128,1,fp8,fp8,0,1.4209334055582683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,64,0,1,float16,fp8,0,1.9416534105936687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,64,128,1,float16,float16,0,1.1667839686075847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,64,128,1,float16,fp8,0,1.1437333424886067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,64,0,1,fp8,fp8,0,1.8271786371866863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,64,0,1,float16,float16,0,2.004858652750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,64,128,1,fp8,fp8,0,1.472490628560384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,64,128,1,float16,float16,0,0.6272000074386597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,64,128,1,float16,fp8,0,0.6147253513336182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,64,0,1,fp8,fp8,0,1.8893013000488281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,64,128,1,fp8,fp8,0,0.7932426929473877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,64,0,1,float16,float16,0,1.0537760257720947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,64,0,1,float16,fp8,0,1.987269401550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,64,0,1,float16,fp8,0,1.0380480289459229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,64,0,1,fp8,fp8,0,0.9896319707234701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,64,128,1,float16,float16,0,0.5212586720784506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,64,128,1,float16,fp8,0,0.5204693476359049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,64,128,1,fp8,fp8,0,0.6852640310923258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,64,0,1,float16,float16,0,0.9434560139973959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,64,0,1,float16,fp8,0,0.9377813339233398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,64,128,1,float16,float16,0,0.5398079951604208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,64,0,1,fp8,fp8,0,0.8861227035522461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,64,128,1,float16,fp8,0,0.5294506549835205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,64,128,1,fp8,fp8,0,0.7145546277364095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,64,0,1,float16,float16,0,0.953221321105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,64,0,1,float16,fp8,0,0.9531413714090983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,64,128,1,float16,float16,0,0.5467040141423544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,64,128,1,float16,fp8,0,0.538698673248291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,64,0,1,fp8,fp8,0,0.9039306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,64,0,1,float16,float16,0,0.978607972462972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,64,128,1,fp8,fp8,0,0.722330649693807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,64,128,1,float16,float16,0,0.5789279937744141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,64,0,1,float16,fp8,0,0.9731520016988119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,64,0,1,fp8,fp8,0,0.9266026814778646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,64,128,1,float16,fp8,0,0.5610560178756714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,64,0,1,float16,float16,0,1.0019839604695637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,64,128,1,float16,float16,0,0.3188906709353129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,64,128,1,fp8,fp8,0,0.746666669845581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,64,128,1,float16,fp8,0,0.31330132484436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,64,0,1,float16,fp8,0,0.995189348856608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,64,0,1,float16,float16,0,0.537285327911377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,64,0,1,fp8,fp8,0,0.9498559633890787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,64,128,1,fp8,fp8,0,0.40217065811157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,64,128,1,float16,float16,0,0.27347733577092487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,64,0,1,float16,fp8,0,0.535370667775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,64,0,1,fp8,fp8,0,0.492847998936971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,64,128,1,float16,fp8,0,0.27325334151585895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,64,128,1,fp8,fp8,0,0.36770665645599365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,64,0,1,float16,float16,0,0.4993973175684611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,64,128,1,float16,float16,0,0.2765653332074483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,64,0,1,fp8,fp8,0,0.4461866617202759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,64,0,1,float16,fp8,0,0.48896535237630206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,64,128,1,float16,fp8,0,0.2786453366279602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,64,0,1,float16,float16,0,0.5063039859135946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,64,128,1,fp8,fp8,0,0.36876265207926434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,64,128,1,float16,float16,0,0.2810399929682414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,64,0,1,float16,fp8,0,0.5001120169957479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,64,0,1,fp8,fp8,0,0.45653867721557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,64,128,1,float16,fp8,0,0.2783573269844055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,64,0,1,float16,float16,0,0.5058773358662924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,64,128,1,fp8,fp8,0,0.37852799892425537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,64,0,1,float16,fp8,0,0.5037599802017212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,64,0,1,fp8,fp8,0,0.4560319979985555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,64,128,1,float16,float16,0,0.28786667188008624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,64,128,1,float16,fp8,0,0.287445326646169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,64,0,1,float16,float16,0,0.5086079835891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,64,128,1,fp8,fp8,0,0.3863893349965413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,64,128,1,float16,float16,0,0.17077332735061646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,64,128,1,float16,fp8,0,0.17023466030756632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,64,0,1,float16,fp8,0,0.5120746692021688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,64,0,1,float16,float16,0,0.2720639904340108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,64,0,1,fp8,fp8,0,0.47119998931884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,64,128,1,fp8,fp8,0,0.22403200467427573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,64,0,1,float16,float16,0,0.25491732358932495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,64,0,1,float16,fp8,0,0.27591466903686523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,64,0,1,fp8,fp8,0,0.2560960054397583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,64,128,1,float16,float16,0,0.15290133158365884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,64,128,1,float16,fp8,0,0.14965867002805075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,64,128,1,fp8,fp8,0,0.20757333437601724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,64,0,1,float16,fp8,0,0.2490560015042623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,64,0,1,fp8,fp8,0,0.24045334259668985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,64,128,1,float16,float16,0,0.1560479998588562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,64,0,1,float16,float16,0,0.25165865818659466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,64,128,1,float16,fp8,0,0.15165332953135172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,64,128,1,fp8,fp8,0,0.21095999081929526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,64,0,1,float16,fp8,0,0.2537279923756917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,64,0,1,fp8,fp8,0,0.24023467302322388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,64,128,1,float16,float16,0,0.15892799695332846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,64,128,1,float16,fp8,0,0.1551413337389628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,64,0,1,float16,float16,0,0.25805866718292236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,64,0,1,fp8,fp8,0,0.2462079922358195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,64,128,1,fp8,fp8,0,0.21053866545359293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,64,0,1,float16,fp8,0,0.25729066133499146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,64,128,1,float16,float16,0,0.1612213353315989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,64,128,1,float16,fp8,0,0.15990933775901794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,64,0,1,float16,float16,0,0.26286399364471436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,64,128,1,fp8,fp8,0,0.2174826661745707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,64,0,1,float16,fp8,0,0.2624906698862712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,64,128,1,float16,fp8,0,0.10140267014503479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,64,128,1,float16,float16,0,0.1023520032564799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,64,0,1,fp8,fp8,0,0.24709333976109824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,64,0,1,float16,float16,0,0.14761066436767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,64,128,1,fp8,fp8,0,0.13716266552607217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,64,0,1,float16,fp8,0,0.15017066399256387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,64,0,1,fp8,fp8,0,0.14405866463979086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,64,128,1,float16,float16,0,0.09087999661763509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,64,0,1,float16,float16,0,0.14153066277503967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,64,128,1,float16,fp8,0,0.09099200367927551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,64,128,1,fp8,fp8,0,0.1190826694170634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,64,0,1,float16,fp8,0,0.1409173309803009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,64,0,1,fp8,fp8,0,0.13371200362841287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,64,128,1,float16,float16,0,0.092357337474823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,64,0,1,float16,float16,0,0.13893866539001465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,64,128,1,float16,fp8,0,0.09190932909647624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,64,128,1,fp8,fp8,0,0.12380799651145935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,64,0,1,float16,fp8,0,0.13936533530553183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,64,0,1,fp8,fp8,0,0.1351093351840973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,64,128,1,float16,float16,0,0.09513599673906963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,64,0,1,float16,float16,0,0.14050133029619852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,64,128,1,float16,fp8,0,0.09305066863695781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,64,128,1,fp8,fp8,0,0.12763200203577676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,64,0,1,float16,fp8,0,0.1407360037167867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,64,0,1,fp8,fp8,0,0.1353386640548706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,64,128,1,fp8,fp8,0,0.13057600458463034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,64,128,1,float16,float16,0,0.09758399923642476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,64,0,1,float16,float16,0,0.1444000005722046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,64,128,1,float16,fp8,0,0.0957493285338084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,64,0,1,float16,fp8,0,0.14378133416175842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,64,0,1,fp8,fp8,0,0.13834666212399802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,64,128,1,float16,float16,0,0.07145066559314728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,64,0,1,float16,float16,0,0.09528533617655437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,64,128,1,float16,fp8,0,0.07086400190989177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,64,0,1,float16,fp8,0,0.09525332848230998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,64,128,1,fp8,fp8,0,0.09403199950853984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,64,0,1,fp8,fp8,0,0.09111467003822327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,64,128,1,float16,float16,0,0.06888000170389812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,64,0,1,float16,float16,0,0.09417066971460979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,64,128,1,float16,fp8,0,0.06894400219122569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,64,128,1,fp8,fp8,0,0.08591999610265096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,64,0,1,float16,fp8,0,0.09501333038012187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,64,0,1,fp8,fp8,0,0.09082133571306865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,64,128,1,float16,float16,0,0.06977066894372304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,64,0,1,float16,float16,0,0.09345600008964539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,64,128,1,float16,fp8,0,0.06894933183987935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,64,128,1,fp8,fp8,0,0.08509332935015361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,64,0,1,float16,fp8,0,0.09419199824333191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,64,0,1,fp8,fp8,0,0.0895146628220876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,64,128,1,float16,float16,0,0.06965866684913635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,64,0,1,float16,float16,0,0.09485333164532979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,64,128,1,fp8,fp8,0,0.08509332935015361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,64,128,1,float16,fp8,0,0.06992533306280772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,64,0,1,float16,fp8,0,0.0939520001411438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,64,0,1,fp8,fp8,0,0.09013332923253377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,64,128,1,float16,float16,0,0.0687306672334671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,64,0,1,float16,float16,0,0.0942186713218689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,64,128,1,float16,fp8,0,0.06937600175539653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,64,128,1,fp8,fp8,0,0.08870399991671245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,64,0,1,float16,fp8,0,0.09445333480834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,64,0,1,fp8,fp8,0,0.09082667032877605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,64,128,1,float16,float16,0,1.401920000712077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,64,128,1,float16,fp8,0,1.387397289276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,64,128,1,fp8,fp8,0,1.756864070892334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,64,0,1,float16,float16,0,2.0143200556437173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,64,128,1,float16,float16,0,1.4118720690409343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,64,0,1,float16,fp8,0,1.9930826822916667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,64,0,1,fp8,fp8,0,1.8367093404134114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,64,128,1,float16,fp8,0,1.3947572708129883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,64,0,1,float16,float16,0,2.0269333521525064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,64,128,1,fp8,fp8,0,1.778586705525716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,64,128,1,float16,float16,0,1.4496266047159831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,64,0,1,fp8,fp8,0,1.8602773348490398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,64,0,1,float16,fp8,0,2.0078773498535156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,64,0,1,float16,float16,0,2.057578722635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,64,128,1,float16,fp8,0,1.4213120142618816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,64,128,1,fp8,fp8,0,1.799722671508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,64,128,1,float16,float16,0,1.5106719334920247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,64,0,1,fp8,fp8,0,1.8786986668904622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,64,0,1,float16,fp8,0,2.0274507204691568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,64,128,1,float16,fp8,0,1.4774400393168132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,64,0,1,float16,float16,0,2.1207733154296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,64,128,1,fp8,fp8,0,1.8506666819254558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,64,128,1,float16,float16,0,0.821061372756958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,64,128,1,float16,fp8,0,0.8010666370391846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,64,0,1,float16,float16,0,1.127562681833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,64,0,1,float16,fp8,0,2.073200066884359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,64,128,1,fp8,fp8,0,1.0277173519134521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,64,0,1,fp8,fp8,0,1.9285173416137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,64,0,1,float16,fp8,0,1.10753067334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,64,128,1,float16,float16,0,0.6874826749165853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,64,0,1,fp8,fp8,0,1.065392017364502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,64,0,1,float16,float16,0,1.0048106511433919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,64,128,1,float16,fp8,0,0.672661304473877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,64,128,1,fp8,fp8,0,0.8838079770406088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,64,0,1,float16,fp8,0,0.9835039774576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,64,128,1,float16,float16,0,0.7042240301767985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,64,0,1,fp8,fp8,0,0.9218719800313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,64,128,1,float16,fp8,0,0.6955466270446777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,64,0,1,float16,float16,0,1.0102612972259521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,64,128,1,fp8,fp8,0,0.9142879645029703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,64,0,1,fp8,fp8,0,0.9487146536509196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,64,0,1,float16,fp8,0,0.9994026819864908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,64,128,1,float16,float16,0,0.7191253503163656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,64,128,1,float16,fp8,0,0.7164533138275146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,64,0,1,float16,float16,0,1.02785062789917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,64,128,1,fp8,fp8,0,0.914954662322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,64,0,1,float16,fp8,0,1.015221357345581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,64,0,1,fp8,fp8,0,0.9628746509552002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,64,128,1,float16,float16,0,0.753007968266805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,64,128,1,float16,fp8,0,0.7319466272989908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,64,0,1,float16,float16,0,1.0500426292419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,64,128,1,fp8,fp8,0,0.952672004699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,64,128,1,float16,float16,0,0.41555734475453693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,64,0,1,fp8,fp8,0,0.9887519677480062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,64,0,1,float16,float16,0,0.5773066679636637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,64,128,1,float16,fp8,0,0.41076799233754474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,64,0,1,float16,fp8,0,1.041541337966919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,64,128,1,fp8,fp8,0,0.5160959959030151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,64,0,1,float16,fp8,0,0.5575253168741862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,64,128,1,float16,fp8,0,0.3511626720428467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,64,128,1,float16,float16,0,0.3501226504643758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,64,0,1,fp8,fp8,0,0.5346506834030151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,64,0,1,float16,float16,0,0.5169920126597086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,64,128,1,fp8,fp8,0,0.4697333176930745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,64,0,1,float16,fp8,0,0.5041919946670532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,64,0,1,fp8,fp8,0,0.47322134176890057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,64,128,1,float16,float16,0,0.3535573482513428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,64,128,1,float16,fp8,0,0.35157867272694904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,64,0,1,float16,float16,0,0.5197226603825887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,64,128,1,fp8,fp8,0,0.4699733257293701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,64,128,1,float16,float16,0,0.3622080087661743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,64,0,1,float16,fp8,0,0.5159680048624674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,64,0,1,fp8,fp8,0,0.4831626812616984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,64,128,1,float16,fp8,0,0.364954670270284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,64,0,1,float16,float16,0,0.5143359899520874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,64,128,1,fp8,fp8,0,0.4752746820449829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,64,128,1,float16,float16,0,0.3738400141398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,64,0,1,float16,fp8,0,0.5132373174031576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,64,128,1,float16,fp8,0,0.37090134620666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,64,0,1,fp8,fp8,0,0.4894453287124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,64,0,1,float16,float16,0,0.5346186558405558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,64,0,1,float16,fp8,0,0.532805323600769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,64,128,1,fp8,fp8,0,0.49928534030914307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,64,128,1,float16,float16,0,0.216703991095225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,64,0,1,fp8,fp8,0,0.49936532974243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,64,128,1,float16,fp8,0,0.21443732579549155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,64,0,1,float16,float16,0,0.29635733366012573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,64,128,1,fp8,fp8,0,0.2781013250350952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,64,0,1,fp8,fp8,0,0.27451199293136597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,64,0,1,float16,fp8,0,0.2980746626853943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,64,128,1,float16,float16,0,0.19377599159876505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,64,0,1,float16,float16,0,0.2714879910151164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,64,128,1,float16,fp8,0,0.18665067354838052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,64,128,1,fp8,fp8,0,0.25522667169570923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,64,0,1,float16,fp8,0,0.2681013345718384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,64,128,1,fp8,fp8,0,0.2555413246154785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,64,0,1,fp8,fp8,0,0.24663466215133667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,64,128,1,float16,float16,0,0.19475199778874716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,64,0,1,float16,float16,0,0.27643199761708576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,64,128,1,float16,fp8,0,0.19172799587249756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,64,128,1,float16,fp8,0,0.19292799631754556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,64,0,1,float16,fp8,0,0.2688480019569397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,64,0,1,fp8,fp8,0,0.2497653365135193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,64,128,1,float16,float16,0,0.1949173410733541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,64,0,1,float16,float16,0,0.27590932448705036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,64,128,1,fp8,fp8,0,0.26315200328826904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,64,0,1,float16,fp8,0,0.27858134110768634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,64,0,1,float16,float16,0,0.28100266059239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,64,0,1,fp8,fp8,0,0.25224000215530396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,64,128,1,float16,float16,0,0.19961067040761313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,64,128,1,float16,fp8,0,0.19907732804616293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,64,128,1,fp8,fp8,0,0.26454933484395343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,64,128,1,float16,float16,0,0.12159466743469238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,64,0,1,float16,fp8,0,0.28386133909225464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,64,0,1,fp8,fp8,0,0.2595733404159546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,64,0,1,float16,float16,0,0.15375999609629312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,64,0,1,fp8,fp8,0,0.14867732922236124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,64,128,1,float16,fp8,0,0.12110400199890137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,64,128,1,fp8,fp8,0,0.16148266196250916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,64,0,1,float16,fp8,0,0.15371732910474142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,64,128,1,float16,float16,0,0.10945066809654236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,64,0,1,float16,float16,0,0.14009066422780356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,64,128,1,float16,fp8,0,0.10578133662541707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,64,128,1,fp8,fp8,0,0.14250666896502176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,64,0,1,float16,fp8,0,0.13878933588663736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,64,0,1,fp8,fp8,0,0.1346506675084432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,64,128,1,float16,float16,0,0.10773332913716634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,64,0,1,float16,float16,0,0.14312000075976053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,64,128,1,float16,fp8,0,0.10738133390744527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,64,128,1,fp8,fp8,0,0.1476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,64,0,1,float16,fp8,0,0.14028799533843994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,64,0,1,fp8,fp8,0,0.13582932949066162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,64,128,1,float16,float16,0,0.10984533031781514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,64,0,1,float16,float16,0,0.1418826679388682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,64,128,1,float16,fp8,0,0.11081066727638245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,64,128,1,fp8,fp8,0,0.15068266789118448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,64,0,1,float16,fp8,0,0.14115200440088907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,64,0,1,fp8,fp8,0,0.13895466923713684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,64,128,1,float16,float16,0,0.1130560040473938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,64,0,1,float16,float16,0,0.14511467019716898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,64,128,1,float16,fp8,0,0.11363733808199565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,64,128,1,fp8,fp8,0,0.15553067127863565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,64,0,1,float16,fp8,0,0.14646933476130167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,64,0,1,fp8,fp8,0,0.142277330160141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,64,128,1,float16,float16,0,0.07421866556008656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,64,0,1,float16,float16,0,0.08653333783149719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,64,128,1,float16,fp8,0,0.07374933362007141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,64,0,1,float16,float16,0,0.0820906658967336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,64,128,1,fp8,fp8,0,0.10151466727256775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,64,0,1,float16,fp8,0,0.08929600318272908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,64,0,1,fp8,fp8,0,0.08752000331878662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,64,128,1,float16,float16,0,0.06701333324114482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,64,128,1,float16,fp8,0,0.06748266518115997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,64,128,1,fp8,fp8,0,0.08358933528264363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,64,0,1,float16,fp8,0,0.08237333099047343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,64,0,1,fp8,fp8,0,0.07879999776681264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,64,128,1,float16,float16,0,0.06786666810512543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,64,128,1,float16,fp8,0,0.06768533090750377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,64,0,1,float16,float16,0,0.08293333152929942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,64,128,1,fp8,fp8,0,0.08412266770998637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,64,0,1,float16,fp8,0,0.08206399778525035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,64,0,1,fp8,fp8,0,0.07846400141716003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,64,128,1,float16,float16,0,0.06888533135255177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,64,0,1,float16,float16,0,0.08293866614500682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,64,128,1,float16,fp8,0,0.06871466835339864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,64,128,1,fp8,fp8,0,0.0846666693687439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,64,0,1,float16,fp8,0,0.08423466483751933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,64,0,1,fp8,fp8,0,0.0798826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,64,128,1,float16,float16,0,0.06969066460927327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,64,0,1,float16,float16,0,0.08352532982826233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,64,128,1,float16,fp8,0,0.07062399884064992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,64,128,1,fp8,fp8,0,0.09489599863688152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,64,0,1,float16,fp8,0,0.08429333567619324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,64,0,1,fp8,fp8,0,0.08221333225568135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,64,128,1,float16,float16,0,0.05414933462937673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,64,0,1,float16,float16,0,0.05955733358860016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,64,128,1,float16,fp8,0,0.053898667295773826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,64,128,1,fp8,fp8,0,0.06410133341948192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,64,0,1,float16,fp8,0,0.05982933441797892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,64,0,1,fp8,fp8,0,0.05685866872469584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,64,0,1,fp8,fp8,0,0.05638400216897329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,64,128,1,float16,float16,0,0.053472002347310386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,64,0,1,float16,float16,0,0.058378666639328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,64,128,1,float16,fp8,0,0.0539680023988088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,64,128,1,fp8,fp8,0,0.06386666496594746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,64,0,1,float16,fp8,0,0.058874666690826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,64,128,1,float16,float16,0,0.05354666709899902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,64,0,1,float16,float16,0,0.05921066800753275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,64,128,1,float16,fp8,0,0.05315199991067251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,64,128,1,fp8,fp8,0,0.06414400041103363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,64,0,1,float16,fp8,0,0.05872533222039541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,64,0,1,fp8,fp8,0,0.05699733396371206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,64,128,1,float16,float16,0,0.053344001372655235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,64,0,1,float16,float16,0,0.05904533465703329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,64,128,1,float16,fp8,0,0.05351999898751577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,64,128,1,fp8,fp8,0,0.06412800153096516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,64,0,1,float16,fp8,0,0.05910933514436086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,64,0,1,fp8,fp8,0,0.05718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,64,128,1,float16,float16,0,0.053616002202034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,64,0,1,float16,float16,0,0.05973333120346069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,64,128,1,float16,fp8,0,0.05351999898751577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,64,128,1,fp8,fp8,0,0.0642133355140686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,64,0,1,float16,fp8,0,0.059338668982187905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,64,0,1,fp8,fp8,0,0.057061334451039634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,64,128,1,float16,float16,0,1.0400160153706868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,64,128,1,float16,fp8,0,1.0278346538543701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,64,0,1,float16,float16,0,1.3159039815266926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,64,128,1,fp8,fp8,0,1.3041280110677083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,64,0,1,float16,fp8,0,1.3019893169403076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,64,128,1,float16,float16,0,1.0416320164998372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,64,0,1,fp8,fp8,0,1.1938453515370686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,64,0,1,float16,float16,0,1.326202630996704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,64,128,1,float16,fp8,0,1.0280106862386067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,64,128,1,fp8,fp8,0,1.3335787455240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,64,0,1,float16,fp8,0,1.3101867039998372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,64,128,1,float16,float16,0,1.06932266553243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,64,0,1,fp8,fp8,0,1.2183466752370198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,64,0,1,float16,float16,0,1.3536693255106609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,64,128,1,float16,fp8,0,1.0522560278574626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,64,128,1,fp8,fp8,0,1.3642826080322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,64,128,1,float16,float16,0,1.1144426663716633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,64,0,1,float16,fp8,0,1.3295306364695232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,64,0,1,fp8,fp8,0,1.25273601214091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,64,128,1,float16,fp8,0,1.0947893460591633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,64,0,1,float16,float16,0,1.396506627400716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,64,128,1,fp8,fp8,0,1.4080959955851238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,64,0,1,float16,fp8,0,1.372538725535075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,64,128,1,float16,float16,0,0.6154986619949341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,64,0,1,float16,float16,0,0.7537973721822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,64,128,1,float16,fp8,0,0.6014986832936605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,64,0,1,fp8,fp8,0,1.296122630437215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,64,128,1,fp8,fp8,0,0.7616960207621256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,64,0,1,float16,fp8,0,0.7404959996541342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,64,128,1,float16,float16,0,0.5162933270136515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,64,0,1,fp8,fp8,0,0.7017119725545248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,64,128,1,float16,fp8,0,0.5070240100224813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,64,0,1,float16,float16,0,0.6544053157170614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,64,128,1,fp8,fp8,0,0.669871966044108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,64,128,1,float16,float16,0,0.5267146825790405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,64,0,1,float16,fp8,0,0.6469226678212484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,64,0,1,fp8,fp8,0,0.6064426501592001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,64,0,1,float16,float16,0,0.6661920150121053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,64,128,1,float16,fp8,0,0.5194666783014933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,64,128,1,fp8,fp8,0,0.6798346837361654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,64,0,1,float16,fp8,0,0.6661493380864462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,64,0,1,fp8,fp8,0,0.6204533179601034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,64,128,1,float16,float16,0,0.538266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,64,128,1,float16,fp8,0,0.53766401608785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,64,0,1,float16,float16,0,0.6821653048197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,64,128,1,fp8,fp8,0,0.6937867005666097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,64,0,1,fp8,fp8,0,0.6378346681594849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,64,0,1,float16,fp8,0,0.6771360238393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,64,128,1,float16,float16,0,0.5648853381474813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,64,128,1,float16,fp8,0,0.5569173494974772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,64,0,1,float16,float16,0,0.7011626561482748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,64,128,1,float16,float16,0,0.31778667370478314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,64,128,1,fp8,fp8,0,0.7247412999471029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,64,0,1,float16,float16,0,0.38971201578776044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,64,128,1,float16,fp8,0,0.31172800064086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,64,0,1,float16,fp8,0,0.693733294804891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,64,0,1,fp8,fp8,0,0.6531680027643839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,64,128,1,fp8,fp8,0,0.39980268478393555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,64,0,1,float16,fp8,0,0.37908267974853516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,64,0,1,fp8,fp8,0,0.35936001936594647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,64,128,1,float16,float16,0,0.2722453276316325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,64,0,1,float16,float16,0,0.3438719908396403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,64,128,1,float16,fp8,0,0.26759467522303265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,64,128,1,float16,float16,0,0.27184534072875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,64,128,1,fp8,fp8,0,0.3590879837671916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,64,0,1,float16,fp8,0,0.33691199620564777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,64,0,1,fp8,fp8,0,0.31544532378514606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,64,0,1,float16,float16,0,0.34962133566538495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,64,128,1,float16,fp8,0,0.27216533819834393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,64,128,1,fp8,fp8,0,0.36211200555165607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,64,0,1,float16,fp8,0,0.34220266342163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,64,128,1,float16,fp8,0,0.2774239977200826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,64,0,1,fp8,fp8,0,0.3152906696001689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,64,128,1,float16,float16,0,0.2781226634979248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,64,0,1,float16,float16,0,0.3497813145319621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,64,128,1,fp8,fp8,0,0.3688640197118123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,64,0,1,float16,fp8,0,0.346560001373291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,64,0,1,fp8,fp8,0,0.3240373333295186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,64,128,1,float16,float16,0,0.285535991191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,64,0,1,float16,float16,0,0.3600746790568034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,64,128,1,float16,fp8,0,0.28220266103744507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,64,0,1,float16,float16,0,0.21813867489496866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,64,128,1,fp8,fp8,0,0.37830932935078937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,64,0,1,float16,fp8,0,0.3566559950510661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,64,128,1,float16,float16,0,0.1683786710103353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,64,0,1,fp8,fp8,0,0.3354719877243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,64,128,1,float16,fp8,0,0.16588800152142844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,64,0,1,float16,float16,0,0.1771519978841146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,64,128,1,fp8,fp8,0,0.22025599082310995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,64,0,1,float16,fp8,0,0.21419199307759604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,64,0,1,fp8,fp8,0,0.18756800889968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,64,128,1,float16,float16,0,0.1465226709842682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,64,128,1,float16,fp8,0,0.14461867014567056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,64,128,1,fp8,fp8,0,0.20111999909083048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,64,0,1,float16,fp8,0,0.17284266153971353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,64,0,1,fp8,fp8,0,0.1692906618118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,64,128,1,float16,float16,0,0.14985600113868713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,64,0,1,float16,fp8,0,0.1738613247871399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,64,0,1,float16,float16,0,0.1754186749458313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,64,128,1,float16,fp8,0,0.15058133006095886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,64,128,1,fp8,fp8,0,0.2013546625773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,64,0,1,fp8,fp8,0,0.1702400048573812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,64,128,1,float16,float16,0,0.1520960032939911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,64,128,1,float16,fp8,0,0.15058666467666626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,64,0,1,float16,float16,0,0.1812266707420349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,64,128,1,fp8,fp8,0,0.20326934258143106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,64,128,1,float16,float16,0,0.15466666221618652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,64,0,1,float16,fp8,0,0.18193066120147705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,64,0,1,fp8,fp8,0,0.1734666625658671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,64,0,1,float16,float16,0,0.18588799238204956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,64,128,1,float16,float16,0,0.09734933574994405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,64,128,1,float16,fp8,0,0.156031996011734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,64,128,1,fp8,fp8,0,0.20874667167663574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,64,0,1,float16,fp8,0,0.18411733706792197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,64,0,1,float16,float16,0,0.10803199807802837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,64,0,1,fp8,fp8,0,0.17718400557835898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,64,128,1,fp8,fp8,0,0.13060266772905985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,64,128,1,float16,fp8,0,0.09635200103123982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,64,0,1,float16,fp8,0,0.10869333148002625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,64,0,1,fp8,fp8,0,0.10549867153167725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,64,128,1,float16,float16,0,0.08456533153851827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,64,0,1,float16,float16,0,0.09604799747467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,64,128,1,float16,fp8,0,0.08288000027338664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,64,128,1,fp8,fp8,0,0.11186666289965312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,64,0,1,float16,fp8,0,0.09520000219345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,64,0,1,fp8,fp8,0,0.09347732861836751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,64,128,1,float16,float16,0,0.0851039985815684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,64,0,1,float16,float16,0,0.09731733798980713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,64,128,1,float16,fp8,0,0.08416000008583069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,64,128,1,fp8,fp8,0,0.11154133081436157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,64,0,1,float16,fp8,0,0.09639466802279155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,64,0,1,fp8,fp8,0,0.09339200456937154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,64,128,1,float16,float16,0,0.08752533793449402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,64,0,1,float16,float16,0,0.09822932879130046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,64,128,1,float16,fp8,0,0.08662933111190796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,64,128,1,fp8,fp8,0,0.11963733037312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,64,0,1,float16,fp8,0,0.09892266988754272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,64,0,1,fp8,fp8,0,0.09722666939099629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,64,128,1,float16,float16,0,0.09139733513196309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,64,0,1,float16,float16,0,0.10103467106819153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,64,128,1,float16,fp8,0,0.08992000420888265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,64,128,1,fp8,fp8,0,0.12486933668454488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,64,0,1,float16,fp8,0,0.10083199540774028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,64,0,1,fp8,fp8,0,0.10063999891281128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,64,128,1,float16,float16,0,0.058335999647776283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,64,0,1,float16,float16,0,0.06341866652170818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,64,128,1,float16,fp8,0,0.059749335050582886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,64,0,1,float16,float16,0,0.06084799766540527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,64,128,1,fp8,fp8,0,0.08360532919565837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,64,0,1,float16,fp8,0,0.0636106679836909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,64,0,1,fp8,fp8,0,0.0634080022573471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,64,128,1,float16,float16,0,0.05426133175690969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,64,128,1,float16,fp8,0,0.05457599957784017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,64,0,1,float16,float16,0,0.0602400004863739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,64,128,1,fp8,fp8,0,0.06990399956703186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,64,0,1,float16,fp8,0,0.0612960010766983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,64,0,1,fp8,fp8,0,0.05856533348560333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,64,128,1,float16,float16,0,0.054560000697771706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,64,128,1,float16,fp8,0,0.054570664962132774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,64,128,1,fp8,fp8,0,0.07111999889214833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,64,0,1,float16,fp8,0,0.06066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,64,0,1,float16,fp8,0,0.06073066592216492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,64,0,1,fp8,fp8,0,0.05849599838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,64,128,1,float16,float16,0,0.054373333851496376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,64,0,1,float16,float16,0,0.061674664417902626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,64,128,1,float16,fp8,0,0.05486933390299479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,64,128,1,fp8,fp8,0,0.07329600056012471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,64,0,1,fp8,fp8,0,0.059258664647738137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,64,128,1,float16,float16,0,0.05625066657861074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,64,0,1,float16,float16,0,0.06127466758092245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,64,128,1,float16,fp8,0,0.05514666438102722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,64,128,1,fp8,fp8,0,0.0711839993794759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,64,0,1,float16,fp8,0,0.06214933097362518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,64,0,1,fp8,fp8,0,0.06006399790445963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,64,128,1,float16,float16,0,0.04764266808827718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,64,0,1,float16,float16,0,0.04654400050640106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,64,128,1,float16,fp8,0,0.04814399778842926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,64,128,1,fp8,fp8,0,0.04985066751639048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,64,0,1,float16,fp8,0,0.04690133531888326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,64,0,1,fp8,fp8,0,0.0436160018046697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,64,128,1,float16,float16,0,0.04541866481304169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,64,0,1,float16,float16,0,0.04601066807905833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,64,128,1,float16,fp8,0,0.043653334180514015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,64,128,1,fp8,fp8,0,0.04952533543109894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,64,0,1,float16,fp8,0,0.045824001232783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,64,0,1,fp8,fp8,0,0.043338666359583534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,64,128,1,float16,float16,0,0.046944002310434975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,64,0,1,float16,float16,0,0.04613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,64,128,1,float16,fp8,0,0.04523199796676636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,64,128,1,fp8,fp8,0,0.05002133548259735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,64,0,1,float16,fp8,0,0.04613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,64,0,1,fp8,fp8,0,0.04343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,64,128,1,float16,float16,0,0.04721599817276001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,64,0,1,float16,float16,0,0.046426668763160706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,64,128,1,float16,fp8,0,0.04790399968624115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,64,128,1,fp8,fp8,0,0.05004266897837321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,64,0,1,float16,fp8,0,0.04615999758243561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,64,0,1,fp8,fp8,0,0.0440586656332016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,64,128,1,float16,float16,0,0.04823466638724009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,64,0,1,float16,float16,0,0.045882667104403176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,64,128,1,float16,fp8,0,0.046096002062161766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,64,128,1,fp8,fp8,0,0.04997866849104563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,64,0,1,float16,fp8,0,0.04632000128428141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,64,0,1,fp8,fp8,0,0.04399466514587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,64,128,1,float16,float16,0,1.2059893608093262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,64,128,1,float16,fp8,0,1.2019039789835613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,64,0,1,float16,float16,0,1.364810625712077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,64,0,1,float16,fp8,0,1.3672266006469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,64,0,1,fp8,fp8,0,1.2720106442769368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,64,128,1,fp8,fp8,0,1.6228532791137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,64,128,1,float16,float16,0,1.216645320256551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,64,0,1,float16,float16,0,1.3825920422871907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,64,128,1,float16,fp8,0,1.219002644220988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,64,128,1,fp8,fp8,0,1.6444106101989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,64,0,1,fp8,fp8,0,1.2839946746826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,64,128,1,float16,float16,0,1.2569226423899333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,64,0,1,float16,fp8,0,1.3732159932454426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,64,0,1,float16,float16,0,1.423327922821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,64,128,1,float16,fp8,0,1.257157325744629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,64,128,1,fp8,fp8,0,1.689253330230713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,64,0,1,float16,fp8,0,1.4376106262207031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,64,0,1,fp8,fp8,0,1.3346667289733887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,64,128,1,float16,float16,0,1.295408010482788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,64,0,1,float16,float16,0,1.4362880388895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,64,128,1,float16,fp8,0,1.2723946571350098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,64,128,1,float16,float16,0,0.7363999684651693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,64,0,1,float16,fp8,0,1.4366559982299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,64,0,1,float16,float16,0,0.8156533241271973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,64,128,1,fp8,fp8,0,1.6917279561360676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,64,0,1,fp8,fp8,0,1.3933653831481934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,64,128,1,float16,fp8,0,0.7148853143056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,64,128,1,fp8,fp8,0,0.9120159943898519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,64,0,1,float16,fp8,0,0.7990933259328207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,64,0,1,fp8,fp8,0,0.7309226989746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,64,128,1,float16,float16,0,0.6174826622009277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,64,0,1,float16,float16,0,0.699669361114502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,64,128,1,float16,fp8,0,0.6172373294830322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,64,0,1,fp8,fp8,0,0.647493322690328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,64,0,1,float16,fp8,0,0.7013653119405111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,64,128,1,fp8,fp8,0,0.8325386842091879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,64,128,1,float16,fp8,0,0.6223946809768677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,64,128,1,float16,float16,0,0.6236159801483154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,64,0,1,float16,float16,0,0.6986080010732015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,64,128,1,fp8,fp8,0,0.8417332967122396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,64,0,1,float16,fp8,0,0.7038880189259847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,64,0,1,fp8,fp8,0,0.6608266830444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,64,128,1,float16,float16,0,0.640554666519165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,64,128,1,float16,fp8,0,0.6361546516418457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,64,0,1,float16,float16,0,0.721893310546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,64,0,1,float16,fp8,0,0.7147946357727051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,64,128,1,fp8,fp8,0,0.8668533166249593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,64,0,1,fp8,fp8,0,0.6860319773356119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,64,128,1,float16,float16,0,0.6518880128860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,64,0,1,float16,float16,0,0.7378400166829427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,64,128,1,float16,fp8,0,0.6442826588948568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,64,0,1,float16,fp8,0,0.7277546723683676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,64,128,1,float16,float16,0,0.3813226620356242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,64,128,1,fp8,fp8,0,0.8564533392588297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,64,0,1,float16,float16,0,0.4211306571960449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,64,0,1,fp8,fp8,0,0.6989440123240153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,64,128,1,float16,fp8,0,0.3704640070597331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,64,128,1,fp8,fp8,0,0.47037867705027264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,64,0,1,float16,fp8,0,0.41353599230448407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,64,0,1,fp8,fp8,0,0.37384533882141113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,64,128,1,float16,float16,0,0.3228000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,64,0,1,float16,float16,0,0.3595999876658122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,64,128,1,float16,fp8,0,0.3228000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,64,128,1,fp8,fp8,0,0.4304320017496745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,64,0,1,float16,fp8,0,0.3592746655146281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,64,0,1,fp8,fp8,0,0.3372480074564616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,64,128,1,float16,float16,0,0.3250986735026042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,64,0,1,float16,float16,0,0.3665066560109456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,64,128,1,fp8,fp8,0,0.43195732434590656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,64,128,1,float16,fp8,0,0.32517866293589276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,64,0,1,float16,fp8,0,0.3659573396046956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,64,0,1,fp8,fp8,0,0.33690134684244794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,64,128,1,float16,float16,0,0.3293973406155904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,64,0,1,float16,float16,0,0.3742773135503133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,64,128,1,float16,fp8,0,0.33377599716186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,64,0,1,float16,fp8,0,0.36912532647450763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,64,128,1,fp8,fp8,0,0.4448053439458211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,64,0,1,fp8,fp8,0,0.34253867467244464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,64,128,1,float16,float16,0,0.339354674021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,64,0,1,float16,float16,0,0.37878398100535077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,64,128,1,float16,fp8,0,0.33419732252756756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,64,128,1,fp8,fp8,0,0.4524266719818115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,64,0,1,float16,fp8,0,0.3803573449452718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,64,128,1,fp8,fp8,0,0.25043733914693195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,64,128,1,float16,float16,0,0.2035199999809265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,64,0,1,fp8,fp8,0,0.35290666421254474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,64,0,1,float16,float16,0,0.22231467564900717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,64,128,1,float16,fp8,0,0.19902400175730386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,64,0,1,float16,fp8,0,0.2182933290799459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,64,0,1,fp8,fp8,0,0.1937546730041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,64,128,1,float16,float16,0,0.17634665966033936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,64,0,1,float16,float16,0,0.1936639944712321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,64,128,1,float16,fp8,0,0.17425066232681274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,64,128,1,fp8,fp8,0,0.23359467585881552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,64,0,1,float16,fp8,0,0.19495999813079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,64,0,1,fp8,fp8,0,0.1755519906679789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,64,128,1,float16,float16,0,0.17531200249989828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,64,0,1,float16,float16,0,0.19828800360361734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,64,128,1,float16,fp8,0,0.1743946671485901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,64,128,1,fp8,fp8,0,0.23613333702087402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,64,0,1,float16,fp8,0,0.1965386668841044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,64,0,1,fp8,fp8,0,0.1785279909769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,64,128,1,float16,float16,0,0.17940799395243326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,64,0,1,float16,float16,0,0.20018666982650757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,64,128,1,float16,fp8,0,0.1808639963467916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,64,128,1,fp8,fp8,0,0.23632532358169556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,64,0,1,float16,fp8,0,0.20171733697255453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,64,0,1,fp8,fp8,0,0.18132267395655313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,64,128,1,float16,float16,0,0.18237332503000894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,64,0,1,float16,float16,0,0.20271466175715128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,64,128,1,float16,fp8,0,0.1801919937133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,64,128,1,fp8,fp8,0,0.2418079972267151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,64,0,1,float16,fp8,0,0.20243734121322632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,64,0,1,fp8,fp8,0,0.1819466749827067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,64,128,1,float16,float16,0,0.11433600385983785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,64,0,1,float16,float16,0,0.12020267049471538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,64,128,1,float16,fp8,0,0.1130506694316864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,64,128,1,fp8,fp8,0,0.14518400033315024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,64,0,1,float16,fp8,0,0.11750400066375732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,64,0,1,fp8,fp8,0,0.10674132903416951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,64,128,1,float16,float16,0,0.09741866588592529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,64,0,1,float16,float16,0,0.10455999771753947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,64,128,1,float16,fp8,0,0.09883733590443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,64,128,1,fp8,fp8,0,0.13351999719937643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,64,0,1,float16,fp8,0,0.102783997853597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,64,0,1,fp8,fp8,0,0.09704533219337463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,64,128,1,float16,float16,0,0.10016533732414246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,64,0,1,float16,float16,0,0.10425600409507751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,64,128,1,float16,fp8,0,0.10085866848627727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,64,128,1,fp8,fp8,0,0.1357439955075582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,64,0,1,float16,fp8,0,0.10520000259081523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,64,0,1,fp8,fp8,0,0.09675733248392741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,64,128,1,float16,float16,0,0.10319999853769939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,64,0,1,float16,float16,0,0.1090186635653178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,64,128,1,float16,fp8,0,0.10148266951243083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,64,128,1,fp8,fp8,0,0.13780267039934793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,64,0,1,float16,fp8,0,0.10815466443697612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,64,0,1,fp8,fp8,0,0.09919466574986775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,64,128,1,float16,float16,0,0.10471999645233154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,64,0,1,float16,float16,0,0.10974400242169698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,64,128,1,float16,fp8,0,0.10511466860771179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,64,128,1,fp8,fp8,0,0.13804266850153604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,64,0,1,float16,fp8,0,0.10967466235160828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,64,0,1,fp8,fp8,0,0.10065600275993347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,64,128,1,float16,float16,0,0.06665066878000896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,64,0,1,float16,float16,0,0.0665280024210612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,64,128,1,float16,fp8,0,0.06563733518123627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,64,128,1,fp8,fp8,0,0.08913066983222961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,64,0,1,float16,float16,0,0.05922133227189382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,64,0,1,float16,fp8,0,0.06557333469390869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,64,0,1,fp8,fp8,0,0.06400533517201741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,64,128,1,float16,float16,0,0.057760000228881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,64,128,1,float16,fp8,0,0.057664001981417336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,64,128,1,fp8,fp8,0,0.0765173335870107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,64,0,1,float16,fp8,0,0.05973866581916809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,64,0,1,fp8,fp8,0,0.05545066793759664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,64,128,1,float16,float16,0,0.05884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,64,0,1,float16,float16,0,0.060122668743133545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,64,128,1,float16,fp8,0,0.059392000238100685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,64,128,1,fp8,fp8,0,0.07533866663773854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,64,0,1,float16,fp8,0,0.06004266440868378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,64,0,1,fp8,fp8,0,0.05724266668160757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,64,128,1,float16,float16,0,0.06028800209363302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,64,0,1,float16,float16,0,0.06131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,64,0,1,float16,float16,0,0.06340266764163971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,64,128,1,float16,fp8,0,0.06011733412742615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,64,128,1,fp8,fp8,0,0.07874666651089986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,64,0,1,float16,fp8,0,0.06294399996598561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,64,0,1,fp8,fp8,0,0.05739733576774597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,64,128,1,float16,float16,0,0.06213866670926412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,64,128,1,float16,fp8,0,0.061205332477887474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,64,128,1,float16,fp8,0,0.0418453315893809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,64,128,1,fp8,fp8,0,0.08203200002511342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,64,0,1,float16,fp8,0,0.0631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,64,0,1,fp8,fp8,0,0.05905599892139435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,64,128,1,float16,float16,0,0.04299733539422353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,64,0,1,float16,float16,0,0.0400693342089653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,64,128,1,fp8,fp8,0,0.051311999559402466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,64,0,1,float16,fp8,0,0.04031999905904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,64,0,1,fp8,fp8,0,0.036320000886917114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,64,128,1,float16,float16,0,0.0383093332250913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,64,0,1,float16,float16,0,0.03731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,64,128,1,float16,fp8,0,0.03909866760174433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,64,128,1,fp8,fp8,0,0.05075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,64,0,1,float16,fp8,0,0.037685332198937736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,64,0,1,float16,fp8,0,0.038047999143600464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,64,0,1,fp8,fp8,0,0.034858666360378265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,64,128,1,float16,float16,0,0.040207999447981514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,64,0,1,float16,float16,0,0.03736533224582672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,64,128,1,float16,fp8,0,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,64,128,1,fp8,fp8,0,0.05003733436266581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,64,0,1,fp8,fp8,0,0.03713600089152654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,64,128,1,float16,float16,0,0.040565334260463715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,64,0,1,float16,float16,0,0.03892799963553747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,64,128,1,float16,fp8,0,0.04087999959786733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,64,128,1,fp8,fp8,0,0.05220800141493479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,64,128,1,float16,fp8,0,0.04095466683308283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,64,0,1,float16,fp8,0,0.03878933439652125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,64,0,1,float16,fp8,0,0.040378667414188385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,64,0,1,fp8,fp8,0,0.03619733452796936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,64,0,1,float16,float16,0,0.02997333308060964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,64,128,1,float16,float16,0,0.04205333193143209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,64,0,1,float16,float16,0,0.04029866556326548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,64,128,1,fp8,fp8,0,0.051327998439470925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,64,0,1,fp8,fp8,0,0.03651199986537298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,64,128,1,float16,float16,0,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,64,128,1,float16,fp8,0,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,64,0,1,float16,float16,0,0.02824000020821889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,64,128,1,fp8,fp8,0,0.03384533276160558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,64,0,1,float16,fp8,0,0.030063999195893604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,64,0,1,fp8,fp8,0,0.029071999092896778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,64,128,1,float16,float16,0,0.02604266752799352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,64,128,1,float16,fp8,0,0.0264533335963885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,64,128,1,fp8,fp8,0,0.03296533226966858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,64,128,1,float16,fp8,0,0.026736001173655193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,64,0,1,float16,fp8,0,0.028880000114440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,64,0,1,fp8,fp8,0,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,64,0,1,fp8,fp8,0,0.027914665639400482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,64,128,1,float16,float16,0,0.027162666122118633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,64,0,1,float16,float16,0,0.02874133239189784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,64,128,1,fp8,fp8,0,0.03315199911594391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,64,0,1,float16,fp8,0,0.028384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,64,128,1,float16,float16,0,0.02699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,64,0,1,float16,float16,0,0.028965334097544353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,64,128,1,float16,fp8,0,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,64,128,1,fp8,fp8,0,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,64,0,1,float16,fp8,0,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,64,128,1,fp8,fp8,0,0.03365866591533025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,64,0,1,fp8,fp8,0,0.028336000939210255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,64,128,1,float16,float16,0,0.027066667874654133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,64,0,1,float16,float16,0,0.029120000700155895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,64,128,1,float16,fp8,0,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,64,0,1,float16,fp8,0,0.029125332832336426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,64,0,1,fp8,fp8,0,0.028832000990708668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,64,128,1,float16,float16,0,1.148634672164917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,64,0,1,float16,float16,0,1.119605302810669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,64,128,1,float16,fp8,0,1.140602668126424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,64,0,1,float16,fp8,0,1.1342933177947998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,64,0,1,fp8,fp8,0,1.0335946877797444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,64,128,1,fp8,fp8,0,1.5274826685587566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,64,128,1,float16,float16,0,1.170954704284668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,64,0,1,float16,float16,0,1.1405866940816243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,64,128,1,float16,fp8,0,1.1467040379842122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,64,0,1,float16,fp8,0,1.1300586859385173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,64,0,1,fp8,fp8,0,1.0633813540140789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,64,128,1,fp8,fp8,0,1.5521492958068848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,64,128,1,float16,float16,0,1.200437307357788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,64,0,1,float16,float16,0,1.1883359750111897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,64,128,1,float16,fp8,0,1.1959306399027507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,64,0,1,float16,fp8,0,1.1774240334828694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,64,0,1,fp8,fp8,0,1.1176586945851643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,64,128,1,fp8,fp8,0,1.5911146799723308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,64,128,1,float16,float16,0,1.2263200283050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,64,0,1,float16,float16,0,1.2021599610646565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,64,128,1,float16,fp8,0,1.1924266815185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,64,0,1,float16,fp8,0,1.186853329340617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,64,128,1,fp8,fp8,0,1.6338826815287273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,64,128,1,float16,float16,0,0.705941359202067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,64,0,1,float16,float16,0,0.6995519797007242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,64,128,1,float16,fp8,0,0.6907093524932861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,64,0,1,fp8,fp8,0,1.1462133725484211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,64,0,1,float16,fp8,0,0.6727253595987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,64,128,1,fp8,fp8,0,0.853653351465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,64,0,1,fp8,fp8,0,0.5900533199310303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,64,128,1,float16,float16,0,0.5810933510462443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,64,0,1,float16,float16,0,0.5739413499832153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,64,128,1,float16,fp8,0,0.5897066593170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,64,128,1,fp8,fp8,0,0.7860159873962402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,64,0,1,fp8,fp8,0,0.5254666805267334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,64,128,1,float16,float16,0,0.586570660273234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,64,0,1,float16,fp8,0,0.5747359991073608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,64,0,1,float16,float16,0,0.5807093381881714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,64,128,1,float16,fp8,0,0.5854880015055338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,64,128,1,fp8,fp8,0,0.7933013439178467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,64,0,1,float16,fp8,0,0.5816053152084351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,64,0,1,fp8,fp8,0,0.5415039857228597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,64,128,1,float16,float16,0,0.6092906792958578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,64,128,1,float16,fp8,0,0.608245333035787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,64,0,1,float16,float16,0,0.5961173375447592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,64,0,1,float16,fp8,0,0.586677352587382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,64,128,1,fp8,fp8,0,0.8172852993011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,64,0,1,fp8,fp8,0,0.5620053211847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,64,128,1,float16,float16,0,0.6237759987513224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,64,0,1,float16,float16,0,0.6067626476287842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,64,128,1,float16,fp8,0,0.6184853315353394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,64,128,1,float16,float16,0,0.36606399218241376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,64,0,1,float16,float16,0,0.35419734319051105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,64,0,1,fp8,fp8,0,0.570688009262085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,64,0,1,float16,fp8,0,0.6050453186035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,64,128,1,fp8,fp8,0,0.8395413557688395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,64,128,1,float16,fp8,0,0.3542720079421997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,64,128,1,fp8,fp8,0,0.44121599197387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,64,0,1,float16,fp8,0,0.34748268127441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,64,0,1,fp8,fp8,0,0.3086400032043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,64,128,1,float16,float16,0,0.3078560034434001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,64,0,1,float16,float16,0,0.2993653416633606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,64,128,1,float16,fp8,0,0.3083733320236206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,64,128,1,fp8,fp8,0,0.404207984606425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,64,0,1,float16,fp8,0,0.2956106662750244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,64,0,1,fp8,fp8,0,0.2747199932734172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,64,128,1,float16,float16,0,0.3063146670659383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,64,0,1,float16,float16,0,0.30131200949350995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,64,128,1,float16,fp8,0,0.3105013370513916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,64,128,1,fp8,fp8,0,0.40930132071177167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,64,0,1,float16,fp8,0,0.2986186742782593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,64,0,1,fp8,fp8,0,0.2737013300259908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,64,128,1,float16,float16,0,0.3197919925053914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,64,0,1,float16,float16,0,0.3038346568743388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,64,128,1,float16,fp8,0,0.31537065903345746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,64,0,1,float16,fp8,0,0.30480533838272095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,64,128,1,fp8,fp8,0,0.4184266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,64,0,1,fp8,fp8,0,0.28003732363382977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,64,128,1,float16,float16,0,0.32608532905578613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,64,0,1,float16,float16,0,0.316101332505544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,64,128,1,float16,fp8,0,0.31945600112279254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,64,128,1,fp8,fp8,0,0.4288160006205241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,64,0,1,float16,fp8,0,0.3114773432413737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,64,128,1,float16,float16,0,0.19432532787322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,64,0,1,fp8,fp8,0,0.2958773374557495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,64,0,1,float16,float16,0,0.18877333402633667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,64,128,1,float16,fp8,0,0.19014400243759155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,64,128,1,fp8,fp8,0,0.23641065756479898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,64,0,1,float16,fp8,0,0.18359466393788657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,64,128,1,float16,fp8,0,0.16793066263198853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,64,0,1,fp8,fp8,0,0.16342399517695108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,64,128,1,float16,float16,0,0.16616533199946085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,64,0,1,float16,float16,0,0.159770667552948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,64,128,1,fp8,fp8,0,0.22106667359670004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,64,0,1,float16,fp8,0,0.1602186659971873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,64,0,1,fp8,fp8,0,0.14628266294797262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,64,128,1,float16,float16,0,0.17021334171295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,64,0,1,float16,float16,0,0.16290666659673056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,64,128,1,float16,fp8,0,0.16914665699005127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,64,128,1,float16,float16,0,0.17313067118326822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,64,128,1,fp8,fp8,0,0.22258667151133218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,64,0,1,float16,fp8,0,0.16234133640925089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,64,0,1,fp8,fp8,0,0.14709867040316263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,64,0,1,float16,float16,0,0.16796799500783285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,64,128,1,float16,fp8,0,0.17249600092569986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,64,0,1,float16,fp8,0,0.16582399606704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,64,128,1,float16,fp8,0,0.17550933361053467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,64,128,1,fp8,fp8,0,0.22498132785161337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,64,0,1,fp8,fp8,0,0.14900267124176025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,64,128,1,float16,float16,0,0.17494932810465494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,64,0,1,float16,float16,0,0.16870399316151938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,64,128,1,fp8,fp8,0,0.22752533356348673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,64,0,1,float16,float16,0,0.10870933532714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,64,0,1,float16,fp8,0,0.16863999764124551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,64,0,1,fp8,fp8,0,0.15171200037002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,64,128,1,float16,float16,0,0.10889066259066264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,64,128,1,float16,fp8,0,0.10849066575368245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,64,128,1,fp8,fp8,0,0.13733866810798645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,64,0,1,float16,fp8,0,0.1053013304869334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,64,0,1,fp8,fp8,0,0.09319466352462769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,64,128,1,float16,float16,0,0.09319999814033508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,64,0,1,float16,float16,0,0.08957866827646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,64,128,1,fp8,fp8,0,0.12641066312789917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,64,128,1,float16,fp8,0,0.09429333607355754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,64,0,1,float16,fp8,0,0.08919999996821086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,64,0,1,float16,fp8,0,0.09036800265312195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,64,0,1,fp8,fp8,0,0.08217066526412964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,64,128,1,float16,float16,0,0.09551999966303508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,64,0,1,float16,float16,0,0.08973333239555359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,64,128,1,float16,fp8,0,0.09567466378211975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,64,128,1,fp8,fp8,0,0.12731732924779257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,64,0,1,fp8,fp8,0,0.08392000198364258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,64,128,1,float16,float16,0,0.09945066769917806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,64,0,1,float16,float16,0,0.09243200222651164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,64,128,1,float16,fp8,0,0.09847467144330342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,64,128,1,fp8,fp8,0,0.12981333335240683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,64,0,1,float16,fp8,0,0.09223999579747517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,64,0,1,fp8,fp8,0,0.08463999629020691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,64,128,1,float16,float16,0,0.10156266887982686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,64,128,1,float16,float16,0,0.0636106679836909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,64,0,1,float16,float16,0,0.09524800380071004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,64,128,1,float16,fp8,0,0.10190932949384053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,64,128,1,fp8,fp8,0,0.1313866674900055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,64,0,1,float16,fp8,0,0.09332266449928284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,64,0,1,fp8,fp8,0,0.08591999610265096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,64,0,1,float16,float16,0,0.058431997895240784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,64,128,1,float16,fp8,0,0.06324266890684764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,64,128,1,fp8,fp8,0,0.08737066388130188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,64,0,1,float16,fp8,0,0.058287998040517174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,64,0,1,fp8,fp8,0,0.05680533250172933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,64,128,1,float16,float16,0,0.05522666871547699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,64,0,1,float16,float16,0,0.05133866767088572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,64,128,1,float16,fp8,0,0.05566399792830149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,64,128,1,fp8,fp8,0,0.07339733342329662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,64,0,1,float16,fp8,0,0.05198933184146881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,64,0,1,fp8,fp8,0,0.04696533580621084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,64,128,1,float16,float16,0,0.056186666091283165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,64,0,1,float16,float16,0,0.05314666529496511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,64,128,1,float16,fp8,0,0.056218668818473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,64,128,1,fp8,fp8,0,0.07366399963696797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,64,0,1,float16,float16,0,0.05406400064627329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,64,0,1,float16,fp8,0,0.05195199946562449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,64,0,1,fp8,fp8,0,0.04764266808827718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,64,128,1,float16,float16,0,0.0582239975531896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,64,128,1,float16,fp8,0,0.057818666100502014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,64,128,1,fp8,fp8,0,0.07483200232187907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,64,0,1,float16,fp8,0,0.053445334235827126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,64,0,1,fp8,fp8,0,0.04950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,64,128,1,float16,float16,0,0.06005866825580597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,64,0,1,float16,float16,0,0.05605333546797434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,64,128,1,float16,fp8,0,0.06009600063165029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,64,128,1,fp8,fp8,0,0.08297066887219746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,64,0,1,float16,fp8,0,0.05550399919350942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,64,0,1,fp8,fp8,0,0.050101334849993386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,64,128,1,float16,float16,0,0.04171200096607208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,64,0,1,float16,float16,0,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,64,128,1,float16,fp8,0,0.042303999265034996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,64,128,1,fp8,fp8,0,0.051125332713127136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,64,0,1,float16,fp8,0,0.03642133375008901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,64,0,1,fp8,fp8,0,0.03179199993610382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,64,128,1,float16,float16,0,0.03950933367013931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,64,0,1,float16,float16,0,0.032560000816980995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,64,128,1,float16,fp8,0,0.038773333032925926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,64,128,1,fp8,fp8,0,0.050154666105906166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,64,0,1,float16,fp8,0,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,64,0,1,fp8,fp8,0,0.030368000268936157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,64,128,1,float16,float16,0,0.03849066545565923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,64,0,1,float16,float16,0,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,64,128,1,float16,fp8,0,0.039162665605545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,64,128,1,fp8,fp8,0,0.050517335534095764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,64,128,1,float16,fp8,0,0.04125866790612539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,64,0,1,float16,fp8,0,0.034160000582536064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,64,0,1,fp8,fp8,0,0.03067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,64,128,1,float16,float16,0,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,64,0,1,float16,float16,0,0.0346666673819224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,64,128,1,fp8,fp8,0,0.05056533217430115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,64,0,1,float16,fp8,0,0.035349334279696144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,64,0,1,fp8,fp8,0,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,64,128,1,float16,float16,0,0.041946664452552795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,64,0,1,float16,float16,0,0.03535466641187668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,64,128,1,float16,fp8,0,0.04135466615358988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,64,128,1,fp8,fp8,0,0.05054933329423269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,64,0,1,float16,fp8,0,0.0354720006386439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,64,0,1,fp8,fp8,0,0.03211733450492223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,64,128,1,float16,float16,0,0.027679999669392902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,64,0,1,float16,float16,0,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,64,128,1,float16,fp8,0,0.02811199923356374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,64,128,1,fp8,fp8,0,0.03444266567627589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,64,128,1,float16,fp8,0,0.0268053337931633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,64,0,1,float16,fp8,0,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,64,0,1,fp8,fp8,0,0.02446399877468745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,64,128,1,float16,float16,0,0.02606933315594991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,64,0,1,float16,float16,0,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,64,128,1,fp8,fp8,0,0.03257066756486893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,64,0,1,float16,fp8,0,0.024341332415739696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,64,0,1,float16,fp8,0,0.024298667907714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,64,0,1,fp8,fp8,0,0.023376000424226124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,64,128,1,float16,float16,0,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,64,0,1,float16,float16,0,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,64,128,1,float16,fp8,0,0.02679466704527537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,64,128,1,fp8,fp8,0,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,64,0,1,fp8,fp8,0,0.023669332265853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,64,128,1,float16,float16,0,0.02659733345111211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,64,0,1,float16,float16,0,0.024288001159826916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,64,128,1,float16,fp8,0,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,64,128,1,fp8,fp8,0,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,64,0,1,float16,fp8,0,0.02442666639884313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,64,0,1,fp8,fp8,0,0.02388266722361247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,64,128,1,float16,float16,0,0.026698666314284008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,64,0,1,float16,float16,0,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,64,128,1,float16,fp8,0,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,64,128,1,fp8,fp8,0,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,64,128,1,fp8,fp8,0,0.03134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,64,0,1,float16,fp8,0,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,64,0,1,fp8,fp8,0,0.024138666689395905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,64,128,1,float16,float16,0,0.024735999604066212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,64,0,1,float16,float16,0,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,64,0,1,float16,float16,0,0.022602667411168415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,64,128,1,float16,fp8,0,0.024495999018351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,64,0,1,float16,fp8,0,0.022602667411168415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,64,0,1,float16,fp8,0,0.02288000037272771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,64,0,1,fp8,fp8,0,0.021573332448800404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,64,128,1,float16,float16,0,0.024234667420387268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,64,128,1,float16,fp8,0,0.024277334411938984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,64,128,1,fp8,fp8,0,0.030799999833106995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,64,0,1,fp8,fp8,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,64,128,1,float16,float16,0,0.023728000621000927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,64,128,1,float16,float16,0,0.0242399995525678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,64,0,1,float16,float16,0,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,64,128,1,float16,fp8,0,0.024346667031447094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,64,128,1,float16,fp8,0,0.02462933212518692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,64,128,1,fp8,fp8,0,0.030853333572546642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,64,0,1,float16,fp8,0,0.022890667120615642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,64,0,1,fp8,fp8,0,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,64,0,1,float16,float16,0,0.022453332940737408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,64,128,1,fp8,fp8,0,0.03070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,64,0,1,float16,fp8,0,0.022709332406520844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,64,0,1,float16,fp8,0,0.022634667654832203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,64,0,1,fp8,fp8,0,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,64,0,1,fp8,fp8,0,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,64,128,1,float16,float16,0,0.024522667129834492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,64,0,1,float16,float16,0,0.022469334304332733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,64,128,1,float16,fp8,0,0.02492800106604894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,64,128,1,fp8,fp8,0,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,64,128,1,float16,float16,0,0.503386656443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,64,0,1,float16,float16,0,0.4893386761347453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,64,128,1,float16,fp8,0,0.500981330871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,64,0,1,float16,fp8,0,0.48788801829020184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,64,0,1,fp8,fp8,0,0.4508426586786906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,64,128,1,fp8,fp8,0,0.6725920041402181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,64,128,1,float16,float16,0,0.5071786642074585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,64,0,1,float16,float16,0,0.4940906763076782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,64,128,1,float16,fp8,0,0.5064213275909424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,64,128,1,fp8,fp8,0,0.6785759925842285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,64,0,1,float16,fp8,0,0.4941386779149373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,64,0,1,fp8,fp8,0,0.4610346555709839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,64,128,1,float16,float16,0,0.5261333386103312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,64,0,1,float16,float16,0,0.5121440092722574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,64,128,1,float16,fp8,0,0.5240266720453898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,64,128,1,fp8,fp8,0,0.7019840081532797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,64,0,1,float16,fp8,0,0.5115146636962891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,64,0,1,fp8,fp8,0,0.48470401763916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,64,128,1,float16,float16,0,0.5386240084966024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,64,0,1,float16,float16,0,0.5303253332773844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,64,128,1,float16,fp8,0,0.5312853256861368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,64,128,1,float16,float16,0,0.3229066729545593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,64,128,1,fp8,fp8,0,0.7257813612620035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,64,0,1,float16,fp8,0,0.5204000075658163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,64,0,1,fp8,fp8,0,0.49715201059977215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,64,0,1,float16,float16,0,0.3184426625569661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,64,128,1,float16,fp8,0,0.3153653343518575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,64,128,1,fp8,fp8,0,0.3722240130106608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,64,0,1,float16,fp8,0,0.30796800057093304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,64,0,1,fp8,fp8,0,0.26049067576726276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,64,128,1,float16,float16,0,0.26044267416000366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,64,0,1,float16,float16,0,0.2542506655057271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,64,128,1,fp8,fp8,0,0.3472213347752889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,64,0,1,float16,fp8,0,0.25539199511210126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,64,128,1,float16,fp8,0,0.26178133487701416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,64,0,1,fp8,fp8,0,0.2347093423207601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,64,128,1,float16,float16,0,0.2645600040753682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,64,0,1,float16,float16,0,0.2571893334388733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,64,128,1,float16,fp8,0,0.26445333162943524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,64,128,1,fp8,fp8,0,0.34761067231496173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,64,128,1,float16,float16,0,0.2730773289998372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,64,0,1,float16,fp8,0,0.25778667132059735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,64,0,1,fp8,fp8,0,0.24024534225463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,64,0,1,float16,float16,0,0.26845333973566693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,64,128,1,float16,fp8,0,0.27134933074315387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,64,128,1,fp8,fp8,0,0.3508960008621216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,64,0,1,float16,fp8,0,0.2664480010668437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,64,0,1,float16,float16,0,0.277946670850118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,64,0,1,fp8,fp8,0,0.24467732508977255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,64,128,1,float16,float16,0,0.28298133611679077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,64,128,1,float16,fp8,0,0.28034132719039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,64,128,1,fp8,fp8,0,0.3610239823659261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,64,0,1,float16,fp8,0,0.27378666400909424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,64,128,1,float16,float16,0,0.17731734116872153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,64,0,1,fp8,fp8,0,0.25574400027592975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,64,0,1,float16,float16,0,0.17533334096272787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,64,0,1,fp8,fp8,0,0.14310399691263834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,64,128,1,float16,fp8,0,0.17249067624409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,64,128,1,fp8,fp8,0,0.1962933341662089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,64,0,1,float16,fp8,0,0.16973867019017538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,64,128,1,float16,float16,0,0.14134933551152548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,64,0,1,float16,float16,0,0.13826666275660196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,64,128,1,float16,fp8,0,0.1402453382809957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,64,128,1,fp8,fp8,0,0.18332799275716147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,64,0,1,float16,fp8,0,0.1371946632862091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,64,0,1,fp8,fp8,0,0.1295253336429596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,64,128,1,float16,float16,0,0.14408000310262045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,64,0,1,float16,float16,0,0.14083733161290488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,64,128,1,float16,fp8,0,0.14417599638303122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,64,0,1,float16,float16,0,0.14781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,64,128,1,fp8,fp8,0,0.1876159906387329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,64,0,1,float16,fp8,0,0.1409066617488861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,64,0,1,fp8,fp8,0,0.13192533453305563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,64,128,1,float16,float16,0,0.15031466881434122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,64,128,1,float16,float16,0,0.15452266732851663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,64,128,1,float16,fp8,0,0.14940266807874045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,64,128,1,fp8,fp8,0,0.1886026660601298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,64,0,1,float16,fp8,0,0.14723733067512512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,64,0,1,fp8,fp8,0,0.13310933113098145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,64,0,1,float16,float16,0,0.1521013379096985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,64,128,1,float16,fp8,0,0.15341333548227945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,64,0,1,float16,fp8,0,0.15155200163523355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,64,128,1,fp8,fp8,0,0.18867733081181845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,64,0,1,fp8,fp8,0,0.1354986627896627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,64,128,1,float16,float16,0,0.10099732875823975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,64,0,1,float16,float16,0,0.10062932968139648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,64,128,1,float16,fp8,0,0.09929066896438599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,64,128,1,fp8,fp8,0,0.10967466235160828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,64,0,1,float16,fp8,0,0.09763733545939128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,64,0,1,fp8,fp8,0,0.08217599987983704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,64,128,1,float16,float16,0,0.07513066629568736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,64,0,1,float16,float16,0,0.0738560010989507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,64,128,1,float16,fp8,0,0.07545066873232524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,64,128,1,fp8,fp8,0,0.10051733255386353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,64,0,1,float16,fp8,0,0.07357866565386455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,64,0,1,fp8,fp8,0,0.07367999851703644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,64,128,1,float16,float16,0,0.07638933261235555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,64,0,1,float16,float16,0,0.07546666761239369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,64,128,1,float16,fp8,0,0.07649600009123485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,64,128,1,fp8,fp8,0,0.10123200217882793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,64,0,1,float16,fp8,0,0.07553066809972127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,64,0,1,fp8,fp8,0,0.07434133191903432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,64,128,1,fp8,fp8,0,0.1030613382657369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,64,128,1,float16,float16,0,0.08042666812737782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,64,0,1,float16,float16,0,0.07855466504891713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,64,128,1,float16,fp8,0,0.07949333389600118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,64,0,1,float16,fp8,0,0.07793599863847096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,64,0,1,fp8,fp8,0,0.07749333480993907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,64,128,1,float16,float16,0,0.08533333738644917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,64,0,1,float16,float16,0,0.08353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,64,128,1,float16,fp8,0,0.08431999882062276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,64,128,1,fp8,fp8,0,0.10488532980283101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,64,0,1,float16,fp8,0,0.0842186709245046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,64,0,1,fp8,fp8,0,0.0777706652879715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,64,0,1,float16,fp8,0,0.04786666731039683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,64,128,1,float16,float16,0,0.05011733373006185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,64,0,1,float16,float16,0,0.04841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,64,128,1,float16,fp8,0,0.04841599861780802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,64,128,1,fp8,fp8,0,0.06490133206049602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,64,0,1,fp8,fp8,0,0.049957334995269775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,64,128,1,float16,float16,0,0.042352000872294106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,64,0,1,float16,float16,0,0.041365332901477814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,64,128,1,float16,fp8,0,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,64,128,1,fp8,fp8,0,0.056890666484832764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,64,0,1,float16,fp8,0,0.041493333876132965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,64,0,1,fp8,fp8,0,0.04298133154710134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,64,128,1,float16,float16,0,0.04283200204372406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,64,0,1,float16,float16,0,0.04232533276081085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,64,128,1,float16,fp8,0,0.04288533329963684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,64,0,1,float16,float16,0,0.04310933252175649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,64,128,1,fp8,fp8,0,0.05611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,64,0,1,float16,fp8,0,0.042021334171295166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,64,0,1,fp8,fp8,0,0.04388799766699473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,64,128,1,float16,float16,0,0.04409599800904592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,64,128,1,float16,fp8,0,0.04398400088151296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,64,128,1,fp8,fp8,0,0.05842666824658712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,64,0,1,float16,fp8,0,0.042805333932240806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,64,0,1,fp8,fp8,0,0.0441599984963735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,64,128,1,float16,float16,0,0.04598933458328247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,64,0,1,float16,float16,0,0.045706664522488914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,64,128,1,float16,fp8,0,0.04565866788228353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,64,128,1,fp8,fp8,0,0.058917333682378135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,64,0,1,float16,fp8,0,0.045909335215886436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,64,128,1,fp8,fp8,0,0.03681066632270813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,64,0,1,fp8,fp8,0,0.04605866471926371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,64,128,1,float16,float16,0,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,64,0,1,float16,float16,0,0.033471999069054924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,64,128,1,float16,fp8,0,0.034341332813103996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,64,0,1,float16,fp8,0,0.034304000437259674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,64,0,1,fp8,fp8,0,0.03028800090154012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,64,128,1,float16,float16,0,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,64,0,1,float16,float16,0,0.030613332986831665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,64,128,1,float16,fp8,0,0.031018666923046112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,64,128,1,fp8,fp8,0,0.03482133398453394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,64,0,1,float16,fp8,0,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,64,0,1,fp8,fp8,0,0.028143999477227528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,64,128,1,float16,float16,0,0.031167998909950256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,64,128,1,float16,float16,0,0.03242666771014532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,64,0,1,float16,float16,0,0.031125334401925404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,64,128,1,float16,fp8,0,0.03179733455181122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,64,128,1,fp8,fp8,0,0.0354720006386439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,64,0,1,float16,fp8,0,0.03145600110292435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,64,0,1,fp8,fp8,0,0.02889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,64,0,1,float16,float16,0,0.03181333343187968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,64,128,1,float16,fp8,0,0.03215999901294708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,64,128,1,fp8,fp8,0,0.03679466744263967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,64,0,1,float16,fp8,0,0.032111999889214836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,64,0,1,fp8,fp8,0,0.02921066681543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,64,128,1,float16,float16,0,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,64,0,1,float16,float16,0,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,64,128,1,float16,fp8,0,0.03347733368476232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,64,128,1,fp8,fp8,0,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,64,128,1,fp8,fp8,0,0.0359946663180987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,64,0,1,float16,fp8,0,0.033045334120591484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,64,0,1,fp8,fp8,0,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,64,128,1,float16,float16,0,0.021920000513394673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,64,0,1,float16,float16,0,0.02182399978240331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,64,128,1,float16,fp8,0,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,64,0,1,float16,fp8,0,0.021685334543387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,64,0,1,fp8,fp8,0,0.02179733415444692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,64,128,1,float16,float16,0,0.020410666863123577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,64,0,1,float16,float16,0,0.02033599962790807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,64,128,1,float16,fp8,0,0.020773333807786305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,64,128,1,fp8,fp8,0,0.023946667710940044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,64,0,1,float16,fp8,0,0.020143999407688778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,64,0,1,fp8,fp8,0,0.020762667059898376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,64,128,1,float16,float16,0,0.02054399996995926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,64,0,1,float16,float16,0,0.020346666375796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,64,128,1,float16,fp8,0,0.020799999435742695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,64,128,1,fp8,fp8,0,0.025055999557177227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,64,0,1,float16,fp8,0,0.020554666717847187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,64,0,1,fp8,fp8,0,0.02062400057911873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,64,128,1,float16,float16,0,0.02086399992307027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,64,0,1,float16,float16,0,0.02024000013868014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,64,128,1,float16,fp8,0,0.020810666183630627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,64,128,1,fp8,fp8,0,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,64,0,1,float16,fp8,0,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,64,0,1,fp8,fp8,0,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,64,128,1,float16,float16,0,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,64,0,1,float16,float16,0,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,64,128,1,float16,fp8,0,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,64,128,1,fp8,fp8,0,0.024698667228221893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,64,0,1,float16,fp8,0,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,64,0,1,fp8,fp8,0,0.021920000513394673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,64,128,1,float16,float16,0,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,64,0,1,float16,float16,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,64,128,1,float16,fp8,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,64,0,1,float16,float16,0,0.016629333297411602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,64,128,1,fp8,fp8,0,0.023024000227451324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,64,0,1,float16,fp8,0,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,64,0,1,fp8,fp8,0,0.01958400011062622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,64,128,1,float16,float16,0,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,64,128,1,float16,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,64,128,1,fp8,fp8,0,0.0229066660006841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,64,0,1,float16,fp8,0,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,64,0,1,fp8,fp8,0,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,64,128,1,float16,float16,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,64,0,1,float16,float16,0,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,64,128,1,float16,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,64,128,1,fp8,fp8,0,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,64,0,1,float16,fp8,0,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,64,0,1,fp8,fp8,0,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,64,0,1,float16,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,64,128,1,float16,float16,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,64,0,1,float16,float16,0,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,64,128,1,float16,fp8,0,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,64,128,1,fp8,fp8,0,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,64,0,1,fp8,fp8,0,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,64,128,1,float16,float16,0,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,64,0,1,float16,float16,0,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,64,128,1,float16,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,64,128,1,fp8,fp8,0,0.02311466634273529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,64,0,1,float16,fp8,0,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,64,0,1,fp8,fp8,0,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,64,128,1,float16,float16,0,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,64,0,1,float16,float16,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,64,128,1,float16,fp8,0,0.01624533285697301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,64,128,1,fp8,fp8,0,0.022128000855445862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,64,0,1,float16,fp8,0,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,64,0,1,fp8,fp8,0,0.018485333770513535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,64,128,1,float16,float16,0,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,64,0,1,float16,float16,0,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,64,128,1,float16,fp8,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,64,128,1,fp8,fp8,0,0.022122666239738464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,64,0,1,float16,fp8,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,64,0,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,64,128,1,float16,float16,0,0.015568000574906668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,64,128,1,float16,float16,0,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,64,0,1,float16,float16,0,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,64,128,1,float16,fp8,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,64,128,1,fp8,fp8,0,0.02184533327817917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,64,128,1,fp8,fp8,0,0.022042666872342426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,64,0,1,float16,fp8,0,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,64,0,1,fp8,fp8,0,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,64,0,1,float16,float16,0,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,64,128,1,float16,fp8,0,0.015978666643301647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,64,0,1,float16,fp8,0,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,64,0,1,fp8,fp8,0,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,64,128,1,float16,float16,0,0.01571200042963028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,64,0,1,float16,float16,0,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,64,128,1,float16,fp8,0,0.016362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,64,128,1,fp8,fp8,0,0.022085333863894146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,64,0,1,float16,fp8,0,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,64,0,1,fp8,fp8,0,0.02195200075705846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,64,128,1,float16,float16,0,0.19176000356674194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,64,0,1,float16,float16,0,0.19222933053970337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,64,128,1,float16,fp8,0,0.1914506753285726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,64,128,1,fp8,fp8,0,0.23574399948120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,64,0,1,float16,fp8,0,0.19140799840291342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,64,0,1,fp8,fp8,0,0.2345973253250122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,64,128,1,float16,float16,0,0.1967839996019999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,64,128,1,fp8,fp8,0,0.23265600204467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,64,0,1,float16,float16,0,0.19606399536132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,64,128,1,float16,fp8,0,0.1941546599070231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,64,0,1,float16,fp8,0,0.1941653291384379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,64,0,1,fp8,fp8,0,0.23355199893315634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,64,128,1,float16,float16,0,0.204858660697937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,64,0,1,float16,float16,0,0.20465066035588583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,64,128,1,fp8,fp8,0,0.24838932355244955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,64,0,1,float16,fp8,0,0.20271466175715128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,64,128,1,float16,fp8,0,0.2037973403930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,64,0,1,fp8,fp8,0,0.24970134099324545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,64,128,1,float16,float16,0,0.21388800938924155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,64,0,1,float16,float16,0,0.21406400203704834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,64,128,1,float16,fp8,0,0.20967467625935873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,64,128,1,fp8,fp8,0,0.2563040057818095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,64,0,1,float16,fp8,0,0.2099413275718689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,64,128,1,float16,float16,0,0.14500799775123596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,64,0,1,fp8,fp8,0,0.2568320035934448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,64,0,1,float16,float16,0,0.1446399986743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,64,128,1,float16,fp8,0,0.1407360037167867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,64,128,1,fp8,fp8,0,0.1409066617488861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,64,0,1,float16,fp8,0,0.14056000113487244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,64,0,1,fp8,fp8,0,0.14218133687973022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,64,128,1,float16,float16,0,0.10590933760007222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,64,0,1,float16,float16,0,0.1055626670519511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,64,128,1,float16,fp8,0,0.1055573324362437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,64,128,1,fp8,fp8,0,0.13054933150609335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,64,0,1,float16,float16,0,0.10912533601125081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,64,0,1,float16,fp8,0,0.10539733370145161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,64,0,1,fp8,fp8,0,0.12812266747156778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,64,128,1,float16,float16,0,0.10841600100199382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,64,128,1,float16,fp8,0,0.10781866312026978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,64,128,1,fp8,fp8,0,0.13033066193262735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,64,0,1,float16,fp8,0,0.10778133074442546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,64,0,1,fp8,fp8,0,0.12844266494115195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,64,128,1,float16,float16,0,0.11468799908955891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,64,0,1,float16,float16,0,0.11412266890207927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,64,0,1,fp8,fp8,0,0.13462932904561362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,64,128,1,float16,fp8,0,0.11257066329320271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,64,128,1,float16,float16,0,0.11917333801587422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,64,128,1,fp8,fp8,0,0.13365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,64,0,1,float16,fp8,0,0.11342933773994446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,64,0,1,float16,float16,0,0.11866666873296101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,64,128,1,float16,fp8,0,0.11787199974060059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,64,128,1,fp8,fp8,0,0.13710932930310568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,64,0,1,float16,fp8,0,0.1163093348344167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,64,0,1,fp8,fp8,0,0.13754133383433023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,64,128,1,float16,float16,0,0.08266666531562805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,64,0,1,float16,float16,0,0.08264000217119853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,64,128,1,float16,fp8,0,0.07985599835713704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,64,128,1,fp8,fp8,0,0.08187733093897502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,64,0,1,float16,fp8,0,0.07974933087825775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,64,0,1,fp8,fp8,0,0.08171199758847554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,64,0,1,float16,fp8,0,0.057818666100502014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,64,128,1,float16,float16,0,0.058042665322621666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,64,0,1,float16,float16,0,0.057536001006762184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,64,128,1,float16,fp8,0,0.057589332262674965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,64,128,1,fp8,fp8,0,0.0728053351243337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,64,0,1,fp8,fp8,0,0.07343466579914093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,64,128,1,float16,float16,0,0.059194669127464294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,64,0,1,float16,float16,0,0.05879466732343038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,64,128,1,float16,fp8,0,0.0589279979467392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,64,128,1,fp8,fp8,0,0.07420800129572551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,64,0,1,float16,fp8,0,0.058677335580190025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,64,0,1,fp8,fp8,0,0.0739573339621226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,64,128,1,float16,float16,0,0.0622026671965917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,64,0,1,float16,float16,0,0.06291733185450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,64,128,1,float16,fp8,0,0.0621013343334198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,64,128,1,fp8,fp8,0,0.07726400097211202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,64,0,1,float16,fp8,0,0.062133332093556724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,64,0,1,fp8,fp8,0,0.07681599756081899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,64,128,1,float16,float16,0,0.06784533460934956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,64,0,1,float16,float16,0,0.06837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,64,128,1,float16,fp8,0,0.06730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,64,128,1,fp8,fp8,0,0.077824001510938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,64,0,1,float16,fp8,0,0.06700266897678375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,64,0,1,fp8,fp8,0,0.07810666660467784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,64,128,1,float16,float16,0,0.0408746674656868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,64,0,1,float16,float16,0,0.0410453329483668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,64,128,1,float16,fp8,0,0.03965866565704346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,64,128,1,fp8,fp8,0,0.050442665815353394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,64,0,1,float16,fp8,0,0.03977599988381068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,64,0,1,fp8,fp8,0,0.0503359983364741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,64,128,1,float16,float16,0,0.03356799980004629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,64,0,1,float16,float16,0,0.03365333378314972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,64,128,1,float16,fp8,0,0.03332266708215078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,64,128,1,fp8,fp8,0,0.04288533329963684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,64,0,1,float16,fp8,0,0.03382399926582972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,64,0,1,fp8,fp8,0,0.04320533573627472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,64,128,1,float16,float16,0,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,64,0,1,float16,float16,0,0.03446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,64,128,1,float16,fp8,0,0.0344106654326121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,64,128,1,fp8,fp8,0,0.04460266729195913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,64,0,1,float16,fp8,0,0.03428266694148382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,64,0,1,fp8,fp8,0,0.04362666606903076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,64,128,1,float16,float16,0,0.03537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,64,0,1,float16,float16,0,0.035504000882307686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,64,128,1,float16,fp8,0,0.03515200068553289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,64,128,1,fp8,fp8,0,0.04413333535194397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,64,0,1,float16,fp8,0,0.03573866685231527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,64,0,1,fp8,fp8,0,0.045194665590922035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,64,128,1,float16,float16,0,0.03737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,64,0,1,float16,float16,0,0.03687999894221624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,64,128,1,float16,fp8,0,0.037077332536379494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,64,128,1,fp8,fp8,0,0.046069333950678505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,64,0,1,float16,fp8,0,0.03714666763941447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,64,0,1,fp8,fp8,0,0.04563733438650767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,64,128,1,float16,float16,0,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,64,0,1,float16,float16,0,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,64,128,1,float16,fp8,0,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,64,128,1,fp8,fp8,0,0.02977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,64,0,1,float16,fp8,0,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,64,0,1,fp8,fp8,0,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,64,128,1,float16,float16,0,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,64,0,1,float16,float16,0,0.022821334501107533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,64,128,1,float16,fp8,0,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,64,128,1,fp8,fp8,0,0.0284853329261144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,64,0,1,float16,fp8,0,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,64,0,1,fp8,fp8,0,0.028416000306606293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,64,128,1,float16,float16,0,0.023498666783173878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,64,0,1,float16,float16,0,0.02382933348417282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,64,128,1,float16,fp8,0,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,64,128,1,fp8,fp8,0,0.028927999238173168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,64,0,1,float16,fp8,0,0.023728000621000927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,64,0,1,fp8,fp8,0,0.02845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,64,128,1,float16,float16,0,0.024314666787783306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,64,0,1,float16,float16,0,0.024527999262015026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,64,128,1,float16,fp8,0,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,64,128,1,fp8,fp8,0,0.029002666473388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,64,0,1,float16,fp8,0,0.0249493345618248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,64,0,1,fp8,fp8,0,0.029711998999118805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,64,128,1,float16,float16,0,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,64,0,1,float16,float16,0,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,64,128,1,float16,fp8,0,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,64,128,1,fp8,fp8,0,0.02980799973011017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,64,0,1,float16,fp8,0,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,64,0,1,fp8,fp8,0,0.029487999776999157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,64,128,1,float16,float16,0,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,64,0,1,float16,float16,0,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,64,128,1,float16,fp8,0,0.019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,64,128,1,fp8,fp8,0,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,64,128,1,float16,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,64,0,1,float16,fp8,0,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,64,0,1,fp8,fp8,0,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,64,128,1,float16,float16,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,64,128,1,float16,float16,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,64,0,1,float16,float16,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,64,128,1,fp8,fp8,0,0.020138667275508244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,64,0,1,float16,fp8,0,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,64,0,1,fp8,fp8,0,0.020181333025296528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,64,0,1,fp8,fp8,0,0.019962667177120846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,64,0,1,float16,float16,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,64,128,1,float16,fp8,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,64,0,1,float16,float16,0,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,64,128,1,fp8,fp8,0,0.02073066681623459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,64,0,1,float16,fp8,0,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,64,0,1,float16,fp8,0,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,64,128,1,float16,float16,0,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,64,128,1,float16,fp8,0,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,64,128,1,fp8,fp8,0,0.020901332298914593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,64,0,1,fp8,fp8,0,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,64,128,1,float16,float16,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,64,0,1,float16,float16,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,64,128,1,float16,fp8,0,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,64,128,1,fp8,fp8,0,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,64,0,1,float16,fp8,0,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,64,128,1,fp8,fp8,0,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,64,0,1,fp8,fp8,0,0.021583999196688335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,64,128,1,float16,float16,0,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,64,0,1,float16,float16,0,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,64,128,1,float16,fp8,0,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,64,0,1,float16,fp8,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,64,0,1,fp8,fp8,0,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,64,128,1,float16,float16,0,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,64,0,1,float16,float16,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,64,128,1,float16,fp8,0,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,64,128,1,fp8,fp8,0,0.019434666881958645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,64,0,1,float16,fp8,0,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,64,0,1,fp8,fp8,0,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,64,128,1,float16,float16,0,0.014469332993030548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,64,0,1,float16,float16,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,64,128,1,float16,float16,0,0.01444799949725469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,64,128,1,float16,fp8,0,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,64,128,1,fp8,fp8,0,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,64,0,1,float16,fp8,0,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,64,0,1,fp8,fp8,0,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,64,0,1,float16,float16,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,64,128,1,float16,fp8,0,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,64,128,1,fp8,fp8,0,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,64,0,1,float16,fp8,0,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,64,0,1,fp8,fp8,0,0.019466667125622433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,64,128,1,float16,float16,0,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,64,0,1,float16,float16,0,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,64,128,1,float16,fp8,0,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,64,128,1,fp8,fp8,0,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,64,0,1,float16,fp8,0,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,64,0,1,fp8,fp8,0,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,64,128,1,float16,float16,0,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,64,0,1,float16,float16,0,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,64,128,1,float16,fp8,0,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,64,128,1,fp8,fp8,0,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,64,0,1,float16,fp8,0,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,64,0,1,fp8,fp8,0,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,64,128,1,float16,float16,0,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,64,0,1,float16,float16,0,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,64,128,1,float16,fp8,0,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,64,128,1,fp8,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,64,0,1,float16,fp8,0,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,64,0,1,fp8,fp8,0,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,64,128,1,float16,float16,0,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,64,0,1,float16,float16,0,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,64,128,1,float16,fp8,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,64,128,1,fp8,fp8,0,0.018277333428462345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,64,128,1,fp8,fp8,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,64,0,1,float16,fp8,0,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,64,0,1,fp8,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,64,128,1,float16,float16,0,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,64,0,1,float16,float16,0,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,64,128,1,float16,fp8,0,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,64,0,1,float16,fp8,0,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,64,0,1,fp8,fp8,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,64,128,1,float16,float16,0,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,64,0,1,float16,float16,0,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,64,128,1,float16,fp8,0,0.013573333621025085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,64,128,1,fp8,fp8,0,0.020245333512624104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,64,0,1,float16,fp8,0,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,64,0,1,fp8,fp8,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,64,128,1,float16,float16,0,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,64,0,1,float16,float16,0,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,64,128,1,float16,fp8,0,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,64,128,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,64,0,1,float16,fp8,0,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,64,0,1,fp8,fp8,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,64,128,1,float16,float16,0,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,64,0,1,float16,float16,0,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,64,128,1,float16,float16,0,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,64,128,1,float16,fp8,0,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,64,128,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,64,0,1,float16,fp8,0,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,64,0,1,fp8,fp8,0,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,64,0,1,float16,float16,0,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,64,128,1,float16,fp8,0,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,64,128,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,64,0,1,float16,fp8,0,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,64,0,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,64,128,1,float16,float16,0,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,64,0,1,float16,float16,0,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,64,128,1,float16,fp8,0,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,64,128,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,64,0,1,float16,fp8,0,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,64,0,1,fp8,fp8,0,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,64,128,1,float16,float16,0,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,64,0,1,fp8,fp8,0,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,64,0,1,float16,float16,0,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,64,128,1,float16,fp8,0,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,64,128,1,fp8,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,64,0,1,float16,fp8,0,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,64,128,1,float16,float16,0,0.0988106628259023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,64,0,1,float16,float16,0,0.09853333234786987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,64,128,1,float16,fp8,0,0.09773332873980205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,64,128,1,fp8,fp8,0,0.16751466194788614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,64,0,1,float16,fp8,0,0.09841066598892212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,64,0,1,fp8,fp8,0,0.16939733425776163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,64,128,1,float16,float16,0,0.10042666395505269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,64,0,1,float16,float16,0,0.10016533732414246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,64,128,1,float16,fp8,0,0.09840533137321472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,64,128,1,fp8,fp8,0,0.1680906613667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,64,0,1,float16,fp8,0,0.09891200065612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,64,0,1,fp8,fp8,0,0.16768000523249307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,64,128,1,float16,float16,0,0.10546132922172546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,64,0,1,float16,float16,0,0.10551466544469197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,64,128,1,float16,fp8,0,0.10467200477917989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,64,128,1,fp8,fp8,0,0.1760586698849996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,64,0,1,float16,fp8,0,0.10566400488217671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,64,0,1,fp8,fp8,0,0.1743519902229309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,64,128,1,float16,float16,0,0.11294399698575337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,64,0,1,float16,float16,0,0.1132586697737376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,64,128,1,float16,fp8,0,0.11114133397738139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,64,0,1,fp8,fp8,0,0.17762132485707602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,64,128,1,fp8,fp8,0,0.1808799902598063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,64,0,1,float16,fp8,0,0.1108746627966563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,64,128,1,float16,float16,0,0.07780266801516215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,64,0,1,float16,float16,0,0.07813333471616109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,64,128,1,float16,fp8,0,0.07610666751861572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,64,128,1,fp8,fp8,0,0.10334400335947673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,64,0,1,float16,fp8,0,0.07613333563009898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,64,0,1,fp8,fp8,0,0.10388799508412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,64,128,1,float16,float16,0,0.0547680010398229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,64,0,1,float16,float16,0,0.053914666175842285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,64,128,1,float16,fp8,0,0.05478399991989136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,64,128,1,fp8,fp8,0,0.0937653382619222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,64,0,1,float16,fp8,0,0.05414933462937673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,64,0,1,fp8,fp8,0,0.09355200330416362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,64,128,1,float16,float16,0,0.05596800148487091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,64,128,1,fp8,fp8,0,0.0944160024325053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,64,0,1,float16,float16,0,0.05560533205668131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,64,128,1,float16,fp8,0,0.054416000843048096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,64,0,1,float16,fp8,0,0.05509333312511444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,64,0,1,fp8,fp8,0,0.09373866518338521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,64,128,1,float16,float16,0,0.058517331878344216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,64,0,1,float16,float16,0,0.05906666815280914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,64,128,1,float16,fp8,0,0.058090666929880776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,64,128,1,fp8,fp8,0,0.09821866949399312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,64,0,1,float16,fp8,0,0.058917333682378135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,64,0,1,fp8,fp8,0,0.0976586639881134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,64,128,1,float16,float16,0,0.0632479985555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,64,0,1,float16,float16,0,0.06367999811967213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,64,128,1,float16,fp8,0,0.06282133360703786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,64,128,1,fp8,fp8,0,0.10147733489672343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,64,0,1,float16,fp8,0,0.06261333326498668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,64,0,1,fp8,fp8,0,0.10008533795674641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,64,128,1,float16,float16,0,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,64,0,1,float16,float16,0,0.040005333721637726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,64,128,1,float16,fp8,0,0.03826133410135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,64,128,1,fp8,fp8,0,0.060922667384147644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,64,0,1,float16,fp8,0,0.038480001191298165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,64,0,1,fp8,fp8,0,0.06099733213583628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,64,128,1,float16,float16,0,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,64,0,1,float16,float16,0,0.03190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,64,128,1,float16,fp8,0,0.03246400008598963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,64,128,1,fp8,fp8,0,0.053120002150535583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,64,0,1,float16,fp8,0,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,64,0,1,fp8,fp8,0,0.052890668312708534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,64,128,1,float16,float16,0,0.03303466737270355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,64,0,1,float16,float16,0,0.03282133241494497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,64,128,1,float16,fp8,0,0.03264000018437704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,64,128,1,fp8,fp8,0,0.0543093333641688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,64,0,1,float16,fp8,0,0.03276266654332479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,64,0,1,fp8,fp8,0,0.054373333851496376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,64,128,1,float16,float16,0,0.034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,64,0,1,float16,float16,0,0.03442666679620743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,64,128,1,float16,fp8,0,0.03384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,64,128,1,fp8,fp8,0,0.055904000997543335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,64,0,1,float16,fp8,0,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,64,0,1,fp8,fp8,0,0.0557226687669754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,64,128,1,float16,float16,0,0.03533333291610082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,64,0,1,fp8,fp8,0,0.057855998476346336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,64,0,1,float16,float16,0,0.035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,64,128,1,float16,fp8,0,0.03516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,64,128,1,fp8,fp8,0,0.0577706644932429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,64,0,1,float16,fp8,0,0.03509866694609324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,64,128,1,float16,float16,0,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,64,0,1,float16,float16,0,0.02367999901374181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,64,128,1,float16,fp8,0,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,64,128,1,fp8,fp8,0,0.034917332231998444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,64,0,1,float16,fp8,0,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,64,0,1,fp8,fp8,0,0.03493333359559377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,64,128,1,float16,float16,0,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,64,0,1,float16,float16,0,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,64,128,1,float16,fp8,0,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,64,128,1,fp8,fp8,0,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,64,0,1,float16,fp8,0,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,64,0,1,fp8,fp8,0,0.033701332906881966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,64,128,1,float16,float16,0,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,64,0,1,float16,float16,0,0.02181333303451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,64,128,1,float16,float16,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,64,128,1,float16,fp8,0,0.021589333812395733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,64,128,1,fp8,fp8,0,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,64,0,1,float16,fp8,0,0.022096000611782074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,64,0,1,fp8,fp8,0,0.03409066547950109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,64,0,1,fp8,fp8,0,0.034832000732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,64,0,1,float16,float16,0,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,64,128,1,float16,fp8,0,0.02260799954334895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,64,128,1,fp8,fp8,0,0.03443733354409536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,64,0,1,float16,fp8,0,0.022437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,64,0,1,float16,fp8,0,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,64,128,1,float16,float16,0,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,64,0,1,float16,float16,0,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,64,128,1,float16,float16,0,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,64,128,1,float16,fp8,0,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,64,128,1,fp8,fp8,0,0.03515200068553289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,64,0,1,fp8,fp8,0,0.03475199888149897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,64,0,1,float16,float16,0,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,64,128,1,float16,fp8,0,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,64,128,1,fp8,fp8,0,0.024346667031447094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,64,0,1,float16,fp8,0,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,64,0,1,fp8,fp8,0,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,64,128,1,float16,float16,0,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,64,0,1,float16,float16,0,0.01642666632930438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,64,128,1,float16,fp8,0,0.016565332810084026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,64,128,1,fp8,fp8,0,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,64,128,1,float16,fp8,0,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,64,0,1,float16,fp8,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,64,0,1,fp8,fp8,0,0.023786666492621105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,64,128,1,float16,float16,0,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,64,0,1,float16,float16,0,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,64,128,1,fp8,fp8,0,0.023930666347344715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,64,0,1,float16,fp8,0,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,64,0,1,fp8,fp8,0,0.023951999843120575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,64,128,1,float16,float16,0,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,64,128,1,float16,float16,0,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,64,0,1,float16,float16,0,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,64,128,1,float16,fp8,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,64,128,1,fp8,fp8,0,0.02447466552257538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,64,0,1,float16,fp8,0,0.016762666404247284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,64,0,1,fp8,fp8,0,0.02422933280467987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,64,0,1,float16,float16,0,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,64,128,1,float16,fp8,0,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,64,128,1,fp8,fp8,0,0.02465066562096278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,64,0,1,float16,fp8,0,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,64,0,1,fp8,fp8,0,0.02477866659561793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,64,128,1,float16,float16,0,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,64,0,1,float16,float16,0,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,64,128,1,float16,fp8,0,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,64,128,1,fp8,fp8,0,0.01989866668979327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,64,0,1,float16,fp8,0,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,64,0,1,fp8,fp8,0,0.019968000551064808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,64,128,1,float16,float16,0,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,64,0,1,float16,float16,0,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,64,128,1,float16,fp8,0,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,64,128,1,fp8,fp8,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,64,0,1,float16,fp8,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,64,0,1,fp8,fp8,0,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,64,0,1,fp8,fp8,0,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,64,128,1,float16,float16,0,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,64,0,1,float16,float16,0,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,64,128,1,float16,fp8,0,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,64,128,1,fp8,fp8,0,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,64,128,1,fp8,fp8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,64,0,1,float16,fp8,0,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,64,128,1,float16,float16,0,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,64,0,1,float16,float16,0,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,64,128,1,float16,fp8,0,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,64,0,1,float16,fp8,0,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,64,0,1,fp8,fp8,0,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,64,128,1,float16,float16,0,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,64,0,1,float16,float16,0,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,64,128,1,float16,fp8,0,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,64,128,1,fp8,fp8,0,0.01950399950146675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,64,0,1,float16,fp8,0,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,64,0,1,fp8,fp8,0,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,64,128,1,float16,float16,0,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,64,0,1,float16,float16,0,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,64,128,1,float16,fp8,0,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,64,128,1,fp8,fp8,0,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,64,0,1,float16,fp8,0,0.012543999900420507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,64,0,1,fp8,fp8,0,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,64,128,1,float16,float16,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,64,0,1,float16,float16,0,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,64,128,1,float16,fp8,0,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,64,128,1,fp8,fp8,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,64,0,1,float16,fp8,0,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,64,0,1,fp8,fp8,0,0.018330667167901993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,64,128,1,float16,float16,0,0.012159999459981918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,64,0,1,float16,float16,0,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,64,128,1,float16,fp8,0,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,64,128,1,fp8,fp8,0,0.01815466706951459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,64,0,1,float16,fp8,0,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,64,0,1,fp8,fp8,0,0.018250666558742523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,64,128,1,float16,float16,0,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,64,0,1,float16,float16,0,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,64,128,1,float16,fp8,0,0.012485332787036896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,64,128,1,fp8,fp8,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,64,0,1,float16,fp8,0,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,64,0,1,fp8,fp8,0,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,64,128,1,float16,float16,0,0.012341332932313284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,64,0,1,float16,float16,0,0.012389333297808966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,64,128,1,float16,fp8,0,0.012650666137536367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,64,128,1,fp8,fp8,0,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,64,0,1,float16,fp8,0,0.0124746672809124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,64,0,1,fp8,fp8,0,0.0183999997874101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,64,128,1,float16,float16,0,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,64,0,1,float16,float16,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,64,128,1,float16,fp8,0,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,64,128,1,fp8,fp8,0,0.018165333817402523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,64,0,1,float16,fp8,0,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,64,0,1,fp8,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,64,128,1,float16,float16,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,64,0,1,float16,float16,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,64,128,1,float16,fp8,0,0.012170666207869848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,64,128,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,64,0,1,float16,fp8,0,0.012362666428089142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,64,0,1,fp8,fp8,0,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,64,128,1,float16,float16,0,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,64,0,1,float16,float16,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,64,128,1,float16,fp8,0,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,64,128,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,64,0,1,float16,fp8,0,0.012223999947309494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,64,0,1,fp8,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,64,128,1,float16,float16,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,64,0,1,float16,float16,0,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,64,128,1,float16,fp8,0,0.012080000092585882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,64,128,1,fp8,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,64,0,1,float16,fp8,0,0.012234666695197424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,64,0,1,fp8,fp8,0,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,64,128,1,float16,float16,0,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,64,0,1,float16,float16,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,64,128,1,float16,fp8,0,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,64,128,1,fp8,fp8,0,0.01773333301146825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,64,0,1,float16,fp8,0,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,64,0,1,fp8,fp8,0,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,64,128,1,float16,float16,0,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,64,0,1,fp8,fp8,0,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,64,0,1,float16,float16,0,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,64,128,1,float16,fp8,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,64,128,1,fp8,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,64,0,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,64,128,1,float16,float16,0,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,64,0,1,float16,float16,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,64,128,1,float16,fp8,0,0.012351999680201212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,64,128,1,fp8,fp8,0,0.01773333301146825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,64,0,1,float16,fp8,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,64,128,1,fp8,fp8,0,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,64,0,1,fp8,fp8,0,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,64,128,1,float16,float16,0,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,64,0,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,64,0,1,float16,float16,0,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,64,128,1,float16,fp8,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,64,0,1,float16,fp8,0,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,64,128,1,float16,float16,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,64,0,1,float16,float16,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,64,128,1,float16,fp8,0,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,64,128,1,fp8,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,64,0,1,float16,fp8,0,0.01239466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,64,0,1,fp8,fp8,0,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,64,128,1,float16,float16,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,64,0,1,float16,float16,0,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,64,128,1,float16,fp8,0,0.012223999947309494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,64,128,1,fp8,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,64,0,1,float16,fp8,0,0.01227733368674914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,64,0,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,64,0,1,float16,float16,0,0.05805333455403646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,64,128,1,float16,float16,0,0.058575997749964394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,64,128,1,float16,fp8,0,0.05808533231417338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,64,128,1,fp8,fp8,0,0.13886933525403342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,64,0,1,float16,fp8,0,0.05788266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,64,0,1,fp8,fp8,0,0.1383093297481537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,64,128,1,float16,float16,0,0.05972266693909963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,64,0,1,float16,float16,0,0.06005333364009857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,64,128,1,float16,fp8,0,0.059152002135912575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,64,0,1,float16,fp8,0,0.059104000528653465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,64,128,1,fp8,fp8,0,0.13980266451835632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,64,0,1,fp8,fp8,0,0.13904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,64,128,1,float16,float16,0,0.06282666822274525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,64,0,1,float16,float16,0,0.06237333516279856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,64,128,1,float16,fp8,0,0.062496001521746315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,64,128,1,fp8,fp8,0,0.1434879998366038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,64,0,1,float16,fp8,0,0.061946665247281395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,64,0,1,fp8,fp8,0,0.1434506674607595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,64,128,1,float16,float16,0,0.06852266689141591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,64,0,1,float16,float16,0,0.0688266654809316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,64,128,1,float16,fp8,0,0.06762133538722992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,64,128,1,fp8,fp8,0,0.14587199687957764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,64,0,1,float16,fp8,0,0.06695466736952464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,64,128,1,float16,float16,0,0.038586666186650596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,64,0,1,fp8,fp8,0,0.14537066221237183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,64,0,1,float16,float16,0,0.0383146678407987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,64,128,1,float16,fp8,0,0.036837334434191384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,64,128,1,fp8,fp8,0,0.08432533343633015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,64,0,1,float16,fp8,0,0.037615999579429626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,64,0,1,fp8,fp8,0,0.08473066488901775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,64,128,1,float16,float16,0,0.03334933271010717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,64,0,1,float16,float16,0,0.03355200091997782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,64,128,1,float16,fp8,0,0.033333333830038704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,64,128,1,fp8,fp8,0,0.07598933577537537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,64,0,1,float16,fp8,0,0.0329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,64,0,1,fp8,fp8,0,0.07663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,64,128,1,float16,float16,0,0.033488000432650246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,64,0,1,float16,float16,0,0.033728001018365227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,64,128,1,float16,fp8,0,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,64,128,1,fp8,fp8,0,0.07737066845099132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,64,0,1,float16,fp8,0,0.033471999069054924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,64,0,1,fp8,fp8,0,0.07704000174999237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,64,128,1,float16,float16,0,0.03445333242416382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,64,0,1,float16,float16,0,0.034661332766215004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,64,0,1,fp8,fp8,0,0.0778186668952306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,64,128,1,float16,fp8,0,0.03422933320204417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,64,128,1,fp8,fp8,0,0.07946666578451793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,64,0,1,float16,fp8,0,0.03461866577466329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,64,128,1,fp8,fp8,0,0.08019733428955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,64,128,1,float16,float16,0,0.03613866617282232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,64,0,1,fp8,fp8,0,0.08188266555468242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,64,0,1,float16,float16,0,0.03648533423741659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,64,128,1,float16,fp8,0,0.03543466577927271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,64,0,1,float16,fp8,0,0.03495999922355016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,64,128,1,float16,float16,0,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,64,0,1,float16,float16,0,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,64,128,1,float16,fp8,0,0.023077333966890972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,64,128,1,fp8,fp8,0,0.0489333321650823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,64,0,1,float16,fp8,0,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,64,0,1,fp8,fp8,0,0.04750399788220724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,64,128,1,float16,float16,0,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,64,0,1,float16,float16,0,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,64,128,1,float16,fp8,0,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,64,128,1,fp8,fp8,0,0.04685866832733154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,64,0,1,float16,fp8,0,0.021669333179791767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,64,0,1,fp8,fp8,0,0.046800002455711365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,64,128,1,float16,float16,0,0.02181866765022278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,64,0,1,float16,float16,0,0.02199999988079071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,64,128,1,float16,fp8,0,0.02183466653029124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,64,128,1,fp8,fp8,0,0.0468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,64,0,1,float16,fp8,0,0.022181332111358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,64,0,1,fp8,fp8,0,0.04737600187460581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,64,128,1,float16,float16,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,64,0,1,float16,float16,0,0.022629333039124806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,64,128,1,float16,fp8,0,0.022282667458057404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,64,128,1,fp8,fp8,0,0.048341333866119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,64,0,1,float16,fp8,0,0.02260799954334895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,64,0,1,fp8,fp8,0,0.04724800089995066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,64,128,1,float16,float16,0,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,64,0,1,float16,float16,0,0.022821334501107533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,64,128,1,float16,fp8,0,0.02294933299223582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,64,128,1,fp8,fp8,0,0.04785599807898203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,64,0,1,float16,fp8,0,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,64,0,1,fp8,fp8,0,0.048026666045188904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,64,128,1,float16,float16,0,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,64,0,1,float16,float16,0,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,64,128,1,float16,fp8,0,0.016074666132529575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,64,128,1,fp8,fp8,0,0.030837332208951313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,64,0,1,float16,fp8,0,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,64,0,1,fp8,fp8,0,0.030965333183606465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,64,128,1,float16,float16,0,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,64,0,1,float16,float16,0,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,64,0,1,float16,float16,0,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,64,128,1,float16,fp8,0,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,64,128,1,fp8,fp8,0,0.029711998999118805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,64,0,1,float16,fp8,0,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,64,0,1,fp8,fp8,0,0.029509333272775013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,64,128,1,float16,float16,0,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,64,128,1,float16,fp8,0,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,64,128,1,fp8,fp8,0,0.03054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,64,0,1,float16,fp8,0,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,64,0,1,fp8,fp8,0,0.03028800090154012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,64,128,1,float16,float16,0,0.015978666643301647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,64,0,1,float16,float16,0,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,64,128,1,float16,fp8,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,64,0,1,float16,fp8,0,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,64,128,1,fp8,fp8,0,0.030741333961486816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,64,0,1,fp8,fp8,0,0.03033600002527237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,64,128,1,float16,float16,0,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,64,0,1,float16,float16,0,0.016143999993801117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,64,128,1,float16,fp8,0,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,64,128,1,fp8,fp8,0,0.03051200012365977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,64,128,1,float16,fp8,0,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,64,0,1,float16,fp8,0,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,64,0,1,fp8,fp8,0,0.031290667752424874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,64,128,1,float16,float16,0,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,64,0,1,float16,float16,0,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,64,128,1,fp8,fp8,0,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,64,0,1,float16,fp8,0,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,64,0,1,fp8,fp8,0,0.022815999885400135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,64,128,1,float16,float16,0,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,64,0,1,float16,float16,0,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,64,128,1,float16,fp8,0,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,64,0,1,float16,float16,0,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,64,128,1,fp8,fp8,0,0.02266666789849599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,64,0,1,float16,fp8,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,64,0,1,fp8,fp8,0,0.022874665757020313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,64,128,1,float16,float16,0,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,64,128,1,float16,fp8,0,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,64,128,1,fp8,fp8,0,0.022495999932289124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,64,0,1,float16,fp8,0,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,64,0,1,fp8,fp8,0,0.02253866692384084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,64,128,1,float16,float16,0,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,64,0,1,float16,float16,0,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,64,128,1,float16,fp8,0,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,64,128,1,fp8,fp8,0,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,64,0,1,float16,fp8,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,64,0,1,fp8,fp8,0,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,64,128,1,float16,float16,0,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,64,0,1,float16,float16,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,64,128,1,float16,fp8,0,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,64,128,1,fp8,fp8,0,0.022815999885400135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,64,0,1,float16,fp8,0,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,64,0,1,fp8,fp8,0,0.02294933299223582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,64,128,1,float16,float16,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,64,0,1,float16,float16,0,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,64,128,1,float16,fp8,0,0.012213333199421564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,64,128,1,fp8,fp8,0,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,64,0,1,float16,fp8,0,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,64,0,1,fp8,fp8,0,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,64,128,1,float16,float16,0,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,64,0,1,float16,float16,0,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,64,128,1,float16,fp8,0,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,64,128,1,fp8,fp8,0,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,64,0,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,64,0,1,fp8,fp8,0,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,64,0,1,float16,fp8,0,0.012362666428089142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,64,128,1,float16,float16,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,64,0,1,float16,float16,0,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,64,128,1,float16,fp8,0,0.012250666817029318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,64,128,1,fp8,fp8,0,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,64,0,1,fp8,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,64,128,1,float16,float16,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,64,0,1,float16,float16,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,64,128,1,float16,fp8,0,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,64,128,1,fp8,fp8,0,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,64,0,1,float16,fp8,0,0.012234666695197424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,64,0,1,fp8,fp8,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,64,0,1,float16,float16,0,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,64,128,1,float16,float16,0,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,64,128,1,float16,fp8,0,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,64,128,1,fp8,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,64,0,1,float16,fp8,0,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,64,0,1,float16,fp8,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,64,0,1,fp8,fp8,0,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,64,128,1,float16,float16,0,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,64,0,1,float16,float16,0,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,64,128,1,float16,fp8,0,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,64,128,1,fp8,fp8,0,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,64,0,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,64,0,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,64,128,1,float16,float16,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,64,0,1,float16,float16,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,64,128,1,float16,fp8,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,64,128,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,64,0,1,fp8,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,64,128,1,float16,float16,0,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,64,0,1,float16,float16,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,64,128,1,float16,fp8,0,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,64,128,1,fp8,fp8,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,64,0,1,float16,fp8,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,64,0,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,64,0,1,fp8,fp8,0,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,64,128,1,float16,float16,0,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,64,0,1,float16,float16,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,64,128,1,float16,fp8,0,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,64,128,1,fp8,fp8,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,64,0,1,float16,fp8,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,64,0,1,fp8,fp8,0,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,64,128,1,float16,float16,0,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,64,0,1,float16,float16,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,64,128,1,float16,fp8,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,64,128,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,64,0,1,float16,fp8,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,64,128,1,float16,float16,0,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,64,0,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,64,128,1,float16,fp8,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,64,128,1,fp8,fp8,0,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,64,0,1,float16,fp8,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,64,0,1,fp8,fp8,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,64,128,1,float16,float16,0,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,64,0,1,float16,float16,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,64,128,1,float16,fp8,0,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,64,128,1,fp8,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,64,0,1,float16,fp8,0,0.012026666353146235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,64,0,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,64,128,1,float16,float16,0,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,64,0,1,float16,float16,0,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,64,128,1,float16,fp8,0,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,64,128,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,64,0,1,float16,fp8,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,64,0,1,fp8,fp8,0,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,64,128,1,float16,float16,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,64,0,1,float16,float16,0,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,64,128,1,float16,fp8,0,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,64,128,1,fp8,fp8,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,64,0,1,float16,fp8,0,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,64,0,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,64,128,1,float16,float16,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,64,0,1,float16,float16,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,64,128,1,float16,fp8,0,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,64,128,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,64,0,1,float16,fp8,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,64,0,1,fp8,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,64,128,1,float16,float16,0,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,64,0,1,float16,float16,0,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,64,128,1,float16,fp8,0,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,64,128,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,64,0,1,float16,fp8,0,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,64,0,1,fp8,fp8,0,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,64,128,1,float16,float16,0,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,64,0,1,float16,float16,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,64,128,1,float16,fp8,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,64,128,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,64,0,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,64,0,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,64,128,1,float16,float16,0,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,64,0,1,float16,float16,0,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,64,128,1,float16,fp8,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,64,128,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,64,0,1,fp8,fp8,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,64,128,1,float16,float16,0,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,64,0,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,64,128,1,float16,fp8,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,64,128,1,fp8,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,64,0,1,float16,fp8,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,64,0,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,64,128,1,float16,float16,0,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,64,0,1,float16,float16,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,64,128,1,float16,fp8,0,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,64,128,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,64,0,1,float16,float16,0,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,64,0,1,float16,fp8,0,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,64,0,1,fp8,fp8,0,0.017701332767804463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,64,128,1,float16,float16,0,0.043280000487963356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,64,128,1,float16,fp8,0,0.043103997906049095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,64,128,1,fp8,fp8,0,0.12326932946840923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,64,0,1,float16,fp8,0,0.04344533383846283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,64,128,1,float16,fp8,0,0.043605332573254905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,64,128,1,float16,float16,0,0.04424533247947693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,64,0,1,fp8,fp8,0,0.12330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,64,0,1,float16,float16,0,0.04414933423201243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,64,0,1,float16,fp8,0,0.043375998735427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,64,128,1,fp8,fp8,0,0.12473066647847493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,64,0,1,fp8,fp8,0,0.1246453324953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,64,128,1,float16,float16,0,0.045114666223526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,64,0,1,float16,fp8,0,0.04466133316357931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,64,0,1,float16,float16,0,0.04453866680463155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,64,128,1,float16,fp8,0,0.04471466441949209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,64,128,1,fp8,fp8,0,0.1251413325468699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,64,0,1,fp8,fp8,0,0.12588799993197122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,64,128,1,float16,float16,0,0.046336000164349876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,64,0,1,float16,float16,0,0.04576533536116282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,64,0,1,fp8,fp8,0,0.12760532895723978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,64,128,1,float16,fp8,0,0.046223998069763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,64,128,1,float16,fp8,0,0.027050666511058807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,64,128,1,fp8,fp8,0,0.12615999579429626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,64,0,1,float16,fp8,0,0.04500266909599304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,64,128,1,float16,float16,0,0.02685333291689555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,64,0,1,float16,float16,0,0.02697066714366277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,64,128,1,fp8,fp8,0,0.07077866792678833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,64,0,1,float16,float16,0,0.026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,64,0,1,float16,fp8,0,0.026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,64,128,1,float16,float16,0,0.02659733345111211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,64,0,1,fp8,fp8,0,0.07082133491834004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,64,128,1,float16,fp8,0,0.02699200063943863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,64,128,1,fp8,fp8,0,0.07012266914049785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,64,0,1,float16,fp8,0,0.026608000199000042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,64,0,1,fp8,fp8,0,0.07007466753323872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,64,128,1,float16,float16,0,0.02657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,64,0,1,float16,float16,0,0.026709333062171936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,64,128,1,float16,fp8,0,0.026687999566396076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,64,128,1,fp8,fp8,0,0.0699893335501353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,64,0,1,float16,fp8,0,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,64,0,1,fp8,fp8,0,0.07117333511511485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,64,128,1,float16,float16,0,0.028570666909217834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,64,0,1,float16,float16,0,0.02703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,64,128,1,float16,fp8,0,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,64,128,1,fp8,fp8,0,0.07049066821734111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,64,0,1,float16,fp8,0,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,64,0,1,fp8,fp8,0,0.07189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,64,128,1,float16,float16,0,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,64,0,1,float16,float16,0,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,64,128,1,float16,fp8,0,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,64,0,1,float16,float16,0,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,64,128,1,fp8,fp8,0,0.07130133112271626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,64,0,1,float16,fp8,0,0.027658666173617046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,64,128,1,float16,float16,0,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,64,0,1,fp8,fp8,0,0.0702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,64,128,1,float16,fp8,0,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,64,0,1,float16,float16,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,64,128,1,float16,fp8,0,0.018426666657129925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,64,128,1,fp8,fp8,0,0.043338666359583534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,64,0,1,float16,fp8,0,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,64,0,1,fp8,fp8,0,0.04460800190766653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,64,128,1,float16,float16,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,64,0,1,float16,float16,0,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,64,128,1,fp8,fp8,0,0.042693331837654114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,64,0,1,float16,fp8,0,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,64,0,1,fp8,fp8,0,0.04355733096599579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,64,128,1,float16,float16,0,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,64,128,1,float16,fp8,0,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,64,0,1,float16,fp8,0,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,64,128,1,fp8,fp8,0,0.04397333165009817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,64,0,1,fp8,fp8,0,0.04353066782156626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,64,128,1,float16,float16,0,0.018373332917690277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,64,0,1,float16,float16,0,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,64,128,1,float16,fp8,0,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,64,128,1,fp8,fp8,0,0.04404266675313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,64,0,1,float16,fp8,0,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,64,0,1,fp8,fp8,0,0.04402133325735728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,64,128,1,float16,float16,0,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,64,0,1,float16,float16,0,0.01971199984351794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,64,128,1,float16,fp8,0,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,64,128,1,fp8,fp8,0,0.04330666859944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,64,0,1,float16,fp8,0,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,64,0,1,fp8,fp8,0,0.043706665436426796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,64,128,1,float16,float16,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,64,0,1,float16,float16,0,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,64,128,1,float16,fp8,0,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,64,128,1,fp8,fp8,0,0.02902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,64,0,1,float16,fp8,0,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,64,0,1,fp8,fp8,0,0.029029332101345062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,64,128,1,float16,float16,0,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,64,0,1,float16,float16,0,0.013946666071812311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,64,128,1,float16,fp8,0,0.013978666315476099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,64,0,1,float16,float16,0,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,64,128,1,fp8,fp8,0,0.02884799987077713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,64,0,1,float16,fp8,0,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,64,0,1,fp8,fp8,0,0.02869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,64,128,1,float16,float16,0,0.01393066719174385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,64,128,1,float16,fp8,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,64,0,1,float16,float16,0,0.013866666704416275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,64,128,1,fp8,fp8,0,0.028362666567166645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,64,0,1,float16,fp8,0,0.014442666123310724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,64,0,1,fp8,fp8,0,0.029050665597120922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,64,128,1,float16,float16,0,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,64,128,1,float16,fp8,0,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,64,128,1,fp8,fp8,0,0.028789333999156952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,64,0,1,float16,fp8,0,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,64,0,1,fp8,fp8,0,0.028442665934562683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,64,128,1,float16,float16,0,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,64,0,1,float16,float16,0,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,64,128,1,float16,fp8,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,64,128,1,fp8,fp8,0,0.028938665986061096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,64,0,1,float16,fp8,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,64,0,1,float16,fp8,0,0.012234666695197424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,64,0,1,fp8,fp8,0,0.028858666618665058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,64,128,1,float16,float16,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,64,0,1,float16,float16,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,64,128,1,float16,fp8,0,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,64,128,1,float16,fp8,0,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,64,128,1,fp8,fp8,0,0.022096000611782074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,64,0,1,fp8,fp8,0,0.02229333420594533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,64,128,1,float16,float16,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,64,0,1,float16,float16,0,0.012432000289360682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,64,128,1,fp8,fp8,0,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,64,0,1,float16,fp8,0,0.012495999534924826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,64,0,1,fp8,fp8,0,0.02146133283774058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,64,128,1,float16,float16,0,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,64,0,1,float16,float16,0,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,64,128,1,float16,fp8,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,64,128,1,fp8,fp8,0,0.021935999393463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,64,0,1,float16,fp8,0,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,64,0,1,fp8,fp8,0,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,64,128,1,float16,float16,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,64,0,1,float16,float16,0,0.01232533281048139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,64,128,1,float16,fp8,0,0.012421333541472753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,64,128,1,fp8,fp8,0,0.021840001145998638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,64,0,1,float16,fp8,0,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,64,0,1,fp8,fp8,0,0.02219199885924657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,64,128,1,float16,float16,0,0.012432000289360682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,64,0,1,float16,float16,0,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,64,128,1,float16,fp8,0,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,64,128,1,fp8,fp8,0,0.021802666286627453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,64,0,1,float16,fp8,0,0.012453333785136541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,64,0,1,fp8,fp8,0,0.021829334398110706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,64,128,1,float16,float16,0,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,64,128,1,float16,fp8,0,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,64,128,1,fp8,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,64,0,1,float16,fp8,0,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,64,0,1,fp8,fp8,0,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,64,128,1,float16,float16,0,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,64,0,1,float16,float16,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,64,128,1,float16,fp8,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,64,128,1,fp8,fp8,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,64,0,1,float16,fp8,0,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,64,0,1,fp8,fp8,0,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,64,128,1,float16,float16,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,64,0,1,float16,float16,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,64,128,1,float16,fp8,0,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,64,128,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,64,0,1,float16,fp8,0,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,64,0,1,fp8,fp8,0,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,64,128,1,float16,float16,0,0.012560000022252401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,64,0,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,64,128,1,float16,fp8,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,64,128,1,fp8,fp8,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,64,0,1,float16,fp8,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,64,0,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,64,128,1,float16,float16,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,64,0,1,float16,float16,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,64,128,1,float16,fp8,0,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,64,128,1,fp8,fp8,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,64,0,1,float16,fp8,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,64,0,1,fp8,fp8,0,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,64,128,1,float16,float16,0,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,64,0,1,float16,float16,0,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,64,128,1,float16,fp8,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,64,128,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,64,0,1,float16,fp8,0,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,64,0,1,fp8,fp8,0,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,64,128,1,float16,float16,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,64,0,1,float16,float16,0,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,64,128,1,float16,fp8,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,64,128,1,fp8,fp8,0,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,64,0,1,float16,fp8,0,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,64,0,1,fp8,fp8,0,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,64,128,1,float16,float16,0,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,64,0,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,64,128,1,float16,fp8,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,64,128,1,fp8,fp8,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,64,0,1,float16,fp8,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,64,0,1,fp8,fp8,0,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,64,128,1,float16,float16,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,64,0,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,64,128,1,float16,fp8,0,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,64,128,1,fp8,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,64,0,1,float16,fp8,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,64,0,1,fp8,fp8,0,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,64,128,1,float16,float16,0,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,64,0,1,float16,float16,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,64,128,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,64,128,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,64,0,1,float16,fp8,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,64,0,1,fp8,fp8,0,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,64,0,1,float16,float16,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,64,128,1,float16,float16,0,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,64,128,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,64,128,1,fp8,fp8,0,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,64,0,1,float16,fp8,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,64,0,1,fp8,fp8,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,64,128,1,float16,float16,0,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,64,0,1,float16,float16,0,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,64,128,1,float16,fp8,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,64,128,1,fp8,fp8,0,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,64,0,1,float16,fp8,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,64,0,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,64,128,1,float16,float16,0,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,64,0,1,float16,float16,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,64,128,1,float16,fp8,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,64,128,1,fp8,fp8,0,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,64,0,1,float16,fp8,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,64,0,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,64,128,1,float16,float16,0,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,64,0,1,float16,float16,0,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,64,128,1,float16,fp8,0,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,64,128,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,64,128,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,64,0,1,float16,fp8,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,64,0,1,fp8,fp8,0,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,64,128,1,float16,float16,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,64,0,1,float16,float16,0,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,64,128,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,64,128,1,float16,fp8,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,64,0,1,float16,fp8,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,64,0,1,fp8,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,64,128,1,float16,float16,0,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,64,0,1,float16,float16,0,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,64,0,1,float16,float16,0,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,64,128,1,float16,fp8,0,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,64,128,1,fp8,fp8,0,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,64,0,1,float16,fp8,0,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,64,0,1,fp8,fp8,0,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,64,128,1,float16,float16,0,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,64,128,1,float16,fp8,0,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,64,0,1,float16,fp8,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,64,0,1,fp8,fp8,0,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,64,128,1,float16,float16,0,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,64,0,1,float16,float16,0,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,64,128,1,float16,fp8,0,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,64,128,1,fp8,fp8,0,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,64,0,1,float16,fp8,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,64,0,1,fp8,fp8,0,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,64,128,1,float16,float16,0,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,64,0,1,float16,float16,0,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,64,128,1,float16,fp8,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,64,128,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,64,0,1,float16,fp8,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,64,0,1,fp8,fp8,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,64,128,1,float16,float16,0,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,64,0,1,float16,float16,0,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,64,128,1,float16,fp8,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,64,128,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,64,0,1,float16,fp8,0,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,64,0,1,fp8,fp8,0,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,64,128,1,float16,float16,0,1.121770699818929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,64,128,1,float16,fp8,0,1.1117173035939534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,64,128,1,fp8,fp8,0,1.4213066101074219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,64,128,1,float16,float16,0,1.174554665883382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,64,128,1,float16,fp8,0,1.1654720306396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,64,128,1,fp8,fp8,0,1.4659466743469238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,64,128,1,float16,float16,0,1.2010986804962158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,64,0,1,fp8,fp8,0,6.474693298339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,64,0,1,float16,float16,0,7.230768203735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,64,0,1,float16,fp8,0,7.174538930257161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,64,128,1,float16,fp8,0,1.190831979115804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,64,0,1,float16,float16,0,7.154848098754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,64,128,1,fp8,fp8,0,1.4779946009318035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,64,128,1,float16,float16,0,0.667909304300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,64,0,1,float16,fp8,0,7.1835676829020185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,64,0,1,fp8,fp8,0,6.528117497762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,64,128,1,float16,fp8,0,0.6491893529891968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,64,128,1,fp8,fp8,0,0.8296053409576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,64,128,1,float16,float16,0,0.5606720050175985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,64,128,1,float16,fp8,0,0.5539893309275309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,64,0,1,float16,float16,0,3.747104008992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,64,0,1,float16,float16,0,7.4650828043619795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,64,128,1,fp8,fp8,0,0.7302026748657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,64,0,1,fp8,fp8,0,3.3775574366251626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,64,0,1,float16,fp8,0,3.6761013666788735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,64,0,1,fp8,fp8,0,6.56005859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,64,128,1,float16,float16,0,0.5925493240356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,64,0,1,float16,float16,0,3.610133488972982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,64,0,1,float16,fp8,0,7.514127731323242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,64,128,1,float16,fp8,0,0.591264009475708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,64,128,1,fp8,fp8,0,0.7556906541188558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,64,128,1,float16,float16,0,0.6092906792958578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,64,128,1,float16,fp8,0,0.5934720039367676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,64,0,1,float16,fp8,0,3.7938238779703775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,64,0,1,fp8,fp8,0,3.2957865397135415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,64,128,1,fp8,fp8,0,0.7766400178273519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,64,0,1,float16,float16,0,3.7196693420410156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,64,128,1,float16,float16,0,0.33721601963043213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,64,0,1,fp8,fp8,0,3.295328140258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,64,128,1,float16,fp8,0,0.35309867064158124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,64,0,1,float16,fp8,0,3.6940746307373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,64,128,1,fp8,fp8,0,0.4363306760787964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,64,0,1,float16,float16,0,3.7239198684692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,64,128,1,float16,float16,0,0.3239733378092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,64,0,1,float16,float16,0,1.8832532564798992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,64,128,1,float16,fp8,0,0.3078453342119853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,64,0,1,fp8,fp8,0,3.338634808858236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,64,128,1,fp8,fp8,0,0.41789865493774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,64,0,1,fp8,fp8,0,1.722991943359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,64,0,1,float16,fp8,0,3.920543988545736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,64,0,1,float16,fp8,0,1.9314346313476562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,64,128,1,float16,float16,0,0.30993600686391193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,64,0,1,float16,float16,0,1.865872065226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,64,128,1,float16,fp8,0,0.31196266412734985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,64,128,1,fp8,fp8,0,0.41995731989542645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,64,0,1,fp8,fp8,0,1.6318987210591633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,64,0,1,float16,fp8,0,1.9098453521728516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,64,128,1,float16,float16,0,0.31916799147923786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,64,128,1,float16,fp8,0,0.31807466348012287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,64,0,1,float16,float16,0,1.8765279452006023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,64,128,1,fp8,fp8,0,0.40855999787648517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,64,0,1,float16,fp8,0,1.9013279279073079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,64,128,1,float16,float16,0,0.22570133209228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,64,0,1,fp8,fp8,0,1.6140213012695312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,64,128,1,float16,fp8,0,0.22399999698003134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,64,0,1,float16,float16,0,1.8714133898417156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,64,128,1,fp8,fp8,0,0.27879999081293744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,64,0,1,float16,float16,0,0.9630239804585775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,64,128,1,float16,float16,0,0.22298665841420492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,64,0,1,fp8,fp8,0,1.6364533106486003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,64,0,1,float16,fp8,0,1.913434664408366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,64,128,1,float16,fp8,0,0.22620266675949097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,64,0,1,float16,fp8,0,0.9630666573842367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,64,0,1,fp8,fp8,0,0.8750186761220297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,64,128,1,fp8,fp8,0,0.27669866879781085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,64,128,1,float16,float16,0,0.22644267479578653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,64,0,1,float16,float16,0,0.9782880147298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,64,128,1,float16,fp8,0,0.22323733568191528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,64,0,1,float16,fp8,0,0.9591999848683676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,64,128,1,fp8,fp8,0,0.27771733204523724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,64,0,1,fp8,fp8,0,0.8973600069681803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,64,0,1,float16,float16,0,0.9770987033843994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,64,128,1,float16,float16,0,0.22286933660507202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,64,0,1,fp8,fp8,0,0.897541364034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,64,128,1,float16,fp8,0,0.22673600912094116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,64,0,1,float16,fp8,0,0.9628960291544596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,64,128,1,fp8,fp8,0,0.278602659702301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,64,0,1,float16,float16,0,0.9783413410186768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,64,0,1,fp8,fp8,0,0.8765013217926025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,64,0,1,float16,fp8,0,0.9813973108927408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,64,128,1,float16,float16,0,0.8494773705800375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,64,128,1,float16,fp8,0,0.8331786791483561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,64,128,1,fp8,fp8,0,1.0758079687754314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,64,128,1,float16,float16,0,0.8994399706522623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,64,128,1,float16,fp8,0,0.8769120375315348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,64,128,1,fp8,fp8,0,1.1149919827779133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,64,0,1,float16,float16,0,4.354725201924642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,64,128,1,float16,float16,0,0.92030930519104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,64,0,1,fp8,fp8,0,3.757754643758138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,64,0,1,float16,fp8,0,4.347567876180013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,64,0,1,float16,float16,0,4.307936032613118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,64,128,1,float16,fp8,0,0.9055840174357096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,64,128,1,fp8,fp8,0,1.1208373705546062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,64,0,1,float16,fp8,0,4.192005475362142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,64,128,1,float16,float16,0,0.505893349647522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,64,0,1,fp8,fp8,0,3.8188158671061196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,64,128,1,float16,fp8,0,0.4855039914449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,64,128,1,fp8,fp8,0,0.6365600029627482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,64,128,1,float16,float16,0,0.4253386656443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,64,0,1,float16,float16,0,2.273434638977051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,64,0,1,float16,float16,0,4.387626647949219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,64,128,1,float16,fp8,0,0.4184266726175944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,64,0,1,float16,fp8,0,2.1678454081217446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,64,0,1,fp8,fp8,0,2.006106694539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,64,0,1,float16,fp8,0,4.468762715657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,64,0,1,fp8,fp8,0,3.8425865173339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,64,128,1,fp8,fp8,0,0.5690613190333048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,64,128,1,float16,float16,0,0.4376000165939331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,64,0,1,float16,float16,0,2.116208076477051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,64,128,1,float16,fp8,0,0.4460746844609578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,64,128,1,fp8,fp8,0,0.5618773301442465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,64,128,1,float16,float16,0,0.44648532072703045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,64,0,1,fp8,fp8,0,1.9380906422932942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,64,0,1,float16,fp8,0,2.1478293736775718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,64,128,1,float16,fp8,0,0.43583468596140545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,64,0,1,float16,float16,0,2.131813367207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,64,128,1,fp8,fp8,0,0.5848000049591064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,64,0,1,float16,fp8,0,2.154965400695801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,64,0,1,fp8,fp8,0,1.976853370666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,64,128,1,float16,float16,0,0.25466134150822956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,64,0,1,float16,float16,0,2.152618726094564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,64,128,1,float16,fp8,0,0.2585013310114543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,64,0,1,float16,fp8,0,2.1289547284444175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,64,0,1,fp8,fp8,0,1.9693013827006023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,64,128,1,fp8,fp8,0,0.3445066610972087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,64,0,1,float16,float16,0,1.121509313583374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,64,128,1,float16,fp8,0,0.24648533264795938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,64,128,1,float16,float16,0,0.23934932549794516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,64,0,1,float16,fp8,0,1.1479732990264893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,64,0,1,fp8,fp8,0,0.9830826918284098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,64,128,1,fp8,fp8,0,0.3105493386586507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,64,0,1,float16,float16,0,1.1135679880777996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,64,0,1,fp8,fp8,0,0.9582826296488444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,64,128,1,float16,float16,0,0.2512906591097514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,64,128,1,float16,fp8,0,0.24285866816838583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,64,0,1,float16,fp8,0,1.1151200135548909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,64,128,1,fp8,fp8,0,0.31219200293223065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,64,128,1,float16,float16,0,0.2425653338432312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,64,0,1,float16,float16,0,1.1153919696807861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,64,128,1,float16,fp8,0,0.2473599910736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,64,128,1,fp8,fp8,0,0.31938133637110394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,64,0,1,float16,fp8,0,1.1135786374409993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,64,0,1,fp8,fp8,0,0.978928009668986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,64,128,1,float16,float16,0,0.17500799894332886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,64,128,1,float16,fp8,0,0.1770026683807373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,64,0,1,float16,float16,0,1.135482629140218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,64,128,1,fp8,fp8,0,0.21620800097783408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,64,0,1,float16,float16,0,0.581274668375651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,64,0,1,float16,fp8,0,1.1391039689381917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,64,0,1,fp8,fp8,0,0.9652746518452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,64,128,1,float16,float16,0,0.17412267128626505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,64,0,1,float16,fp8,0,0.5856426556905111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,64,0,1,fp8,fp8,0,0.5357760190963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,64,128,1,float16,fp8,0,0.17454399665196738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,64,0,1,float16,float16,0,0.5852106809616089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,64,128,1,fp8,fp8,0,0.2158400019009908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,64,128,1,float16,float16,0,0.17478400468826294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,64,0,1,float16,fp8,0,0.5894240140914917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,64,0,1,fp8,fp8,0,0.5330026547114054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,64,128,1,float16,fp8,0,0.17564266920089722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,64,0,1,float16,fp8,0,0.5849813222885132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,64,128,1,fp8,fp8,0,0.21529599030812582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,64,0,1,float16,float16,0,0.5842826763788859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,64,128,1,float16,float16,0,0.17572800318400064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,64,0,1,fp8,fp8,0,0.5366079807281494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,64,128,1,float16,fp8,0,0.17575466632843018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,64,128,1,fp8,fp8,0,0.21730132897694907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,64,0,1,float16,float16,0,0.5873279968897501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,64,0,1,float16,fp8,0,0.5850933392842611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,64,0,1,fp8,fp8,0,0.5401013294855753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,64,128,1,float16,float16,0,0.7020586331685384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,64,128,1,float16,fp8,0,0.6990506649017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,64,128,1,fp8,fp8,0,0.9004159768422445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,64,128,1,float16,float16,0,0.746239980061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,64,128,1,float16,fp8,0,0.7393866380055746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,64,128,1,fp8,fp8,0,0.9317440191904703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,64,0,1,float16,float16,0,2.9861599604288735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,64,0,1,fp8,fp8,0,2.7578986485799155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,64,0,1,float16,fp8,0,2.9846986134847007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,64,128,1,float16,float16,0,0.7574293613433838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,64,0,1,float16,float16,0,3.019904136657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,64,128,1,float16,fp8,0,0.7551146348317465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,64,128,1,fp8,fp8,0,0.9473919868469238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,64,0,1,float16,fp8,0,3.00545597076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,64,128,1,float16,float16,0,0.4137333234151204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,64,0,1,fp8,fp8,0,2.7848587036132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,64,128,1,float16,fp8,0,0.40276801586151123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,64,128,1,fp8,fp8,0,0.5267146825790405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,64,0,1,float16,float16,0,3.1357332865397134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,64,0,1,float16,float16,0,1.567189375559489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,64,128,1,float16,float16,0,0.35728001594543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,64,0,1,float16,fp8,0,3.0419092178344727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,64,0,1,fp8,fp8,0,2.779738744099935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,64,0,1,float16,fp8,0,1.557674725850423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,64,128,1,float16,fp8,0,0.352133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,64,0,1,fp8,fp8,0,1.4565547307332356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,64,128,1,fp8,fp8,0,0.4659786621729533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,64,128,1,float16,float16,0,0.36375999450683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,64,128,1,float16,fp8,0,0.3588053385416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,64,0,1,float16,float16,0,1.519877274831136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,64,128,1,fp8,fp8,0,0.48297067483266193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,64,0,1,float16,fp8,0,1.5572320620218914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,64,0,1,fp8,fp8,0,1.3941760063171387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,64,128,1,float16,float16,0,0.37189332644144696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,64,0,1,float16,float16,0,1.5529707272847493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,64,128,1,float16,fp8,0,0.37508265177408856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,64,128,1,fp8,fp8,0,0.4843519926071167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,64,0,1,float16,fp8,0,1.5487732887268066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,64,0,1,fp8,fp8,0,1.4309919675191243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,64,128,1,float16,float16,0,0.22310400009155273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,64,0,1,float16,float16,0,1.5529920260111492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,64,128,1,float16,fp8,0,0.22415467103322348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,64,128,1,fp8,fp8,0,0.2911093235015869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,64,0,1,float16,float16,0,0.8434133529663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,64,0,1,float16,fp8,0,1.5585120519002278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,64,128,1,float16,float16,0,0.20645866791407266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,64,0,1,fp8,fp8,0,1.4462453524271648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,64,128,1,float16,fp8,0,0.20454400777816772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,64,0,1,float16,fp8,0,0.8392319679260254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,64,0,1,fp8,fp8,0,0.7236426671346029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,64,128,1,fp8,fp8,0,0.27180800835291546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,64,128,1,float16,float16,0,0.20948266983032227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,64,0,1,float16,float16,0,0.8238346576690674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,64,128,1,float16,fp8,0,0.206496000289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,64,0,1,fp8,fp8,0,0.7040053208669027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,64,0,1,float16,fp8,0,0.8229173024495443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,64,128,1,fp8,fp8,0,0.2709280053774516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,64,128,1,float16,float16,0,0.21287999550501505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,64,0,1,float16,float16,0,0.8259893258412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,64,128,1,float16,fp8,0,0.20966933170954385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,64,0,1,fp8,fp8,0,0.7125706672668457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,64,128,1,fp8,fp8,0,0.2781333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,64,0,1,float16,fp8,0,0.8301706314086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,64,128,1,float16,float16,0,0.15221333503723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,64,0,1,float16,float16,0,0.8276746273040771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,64,128,1,float16,fp8,0,0.1523413360118866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,64,0,1,float16,fp8,0,0.821664015452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,64,128,1,fp8,fp8,0,0.1889866590499878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,64,0,1,float16,float16,0,0.4288906653722127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,64,0,1,fp8,fp8,0,0.7200799783070883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,64,128,1,float16,float16,0,0.1497760017712911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,64,128,1,fp8,fp8,0,0.18710400660832724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,64,0,1,fp8,fp8,0,0.4023466507593791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,64,0,1,float16,fp8,0,0.433626651763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,64,128,1,float16,fp8,0,0.14983999729156494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,64,0,1,float16,float16,0,0.4309920072555542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,64,128,1,float16,float16,0,0.14990400274594626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,64,0,1,float16,fp8,0,0.4281280040740967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,64,128,1,float16,fp8,0,0.15152532855669656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,64,0,1,fp8,fp8,0,0.4005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,64,0,1,float16,float16,0,0.4317386547724406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,64,128,1,fp8,fp8,0,0.1864853302637736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,64,128,1,float16,float16,0,0.1504746675491333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,64,0,1,float16,fp8,0,0.425493319829305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,64,128,1,float16,fp8,0,0.1504853367805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,64,0,1,fp8,fp8,0,0.39396266142527264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,64,0,1,float16,float16,0,0.428549329439799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,64,128,1,fp8,fp8,0,0.1880693236986796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,64,0,1,float16,fp8,0,0.43193066120147705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,64,0,1,fp8,fp8,0,0.398357351620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,64,128,1,float16,float16,0,1.1243680318196614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,64,128,1,float16,fp8,0,1.0994880199432373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,64,128,1,fp8,fp8,0,1.4007892608642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,64,128,1,float16,float16,0,1.180250644683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,64,128,1,float16,fp8,0,1.1651519934336345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,64,128,1,fp8,fp8,0,1.451189359029134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,64,0,1,float16,float16,0,4.080170631408691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,64,0,1,fp8,fp8,0,3.594245274861654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,64,0,1,float16,fp8,0,4.0371519724528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,64,0,1,float16,float16,0,4.12446943918864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,64,128,1,float16,float16,0,1.1884160041809082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,64,128,1,float16,fp8,0,1.1741173267364502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,64,0,1,float16,fp8,0,4.020096143086751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,64,128,1,fp8,fp8,0,1.5094186464945476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,64,0,1,fp8,fp8,0,3.6378933588663735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,64,128,1,float16,float16,0,0.6552053292592367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,64,128,1,float16,fp8,0,0.6438986857732137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,64,128,1,fp8,fp8,0,0.8106773694356283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,64,0,1,float16,float16,0,4.084447860717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,64,128,1,float16,float16,0,0.5443679889043173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,64,0,1,float16,float16,0,2.162351926167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,64,0,1,fp8,fp8,0,3.672074635823568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,64,0,1,float16,fp8,0,2.064202626546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,64,0,1,float16,fp8,0,4.225818634033203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,64,128,1,float16,fp8,0,0.5363146861394247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,64,0,1,fp8,fp8,0,1.92414395014445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,64,128,1,fp8,fp8,0,0.7312053044637045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,64,128,1,float16,float16,0,0.5974453290303549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,64,128,1,float16,fp8,0,0.5800319910049438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,64,0,1,float16,float16,0,1.9747360547383626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,64,128,1,fp8,fp8,0,0.7630826632181803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,64,0,1,float16,fp8,0,1.9758987426757812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,64,0,1,fp8,fp8,0,1.853216012318929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,64,128,1,float16,float16,0,0.599290649096171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,64,0,1,float16,float16,0,2.0131093660990396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,64,128,1,float16,fp8,0,0.6097866694132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,64,128,1,float16,float16,0,0.33059199651082355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,64,128,1,fp8,fp8,0,0.7745813528696696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,64,0,1,float16,fp8,0,2.0035093625386557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,64,0,1,fp8,fp8,0,1.8550880750020344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,64,0,1,float16,float16,0,2.056063969930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,64,128,1,float16,fp8,0,0.32390934228897095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,64,128,1,fp8,fp8,0,0.42738668123881024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,64,0,1,float16,float16,0,1.073034683863322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,64,128,1,float16,float16,0,0.2977760036786397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,64,0,1,float16,fp8,0,2.050384044647217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,64,0,1,fp8,fp8,0,1.9109172821044922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,64,128,1,float16,fp8,0,0.2915733257929484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,64,0,1,fp8,fp8,0,0.9589013258616129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,64,0,1,float16,fp8,0,1.0528480211893718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,64,128,1,fp8,fp8,0,0.38599999745686847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,64,0,1,float16,float16,0,1.0245760281880696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,64,128,1,float16,float16,0,0.29974399010340375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,64,128,1,float16,fp8,0,0.297818660736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,64,0,1,float16,fp8,0,1.0390666325887044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,64,0,1,fp8,fp8,0,0.9197813669840494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,64,128,1,fp8,fp8,0,0.39290134112040204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,64,0,1,float16,float16,0,1.0494399865468342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,64,128,1,float16,float16,0,0.30450133482615155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,64,128,1,float16,fp8,0,0.3041920065879822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,64,128,1,fp8,fp8,0,0.3928426504135132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,64,128,1,float16,float16,0,0.1824373404184977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,64,0,1,fp8,fp8,0,0.9110453128814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,64,0,1,float16,fp8,0,1.0449919700622559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,64,0,1,float16,float16,0,1.0482453505198162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,64,128,1,fp8,fp8,0,0.23814932505289713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,64,128,1,float16,fp8,0,0.18164799610773721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,64,0,1,float16,float16,0,0.5329333146413168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,64,0,1,fp8,fp8,0,0.9337866306304932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,64,0,1,float16,fp8,0,1.0527466932932537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,64,0,1,float16,fp8,0,0.5261919895807902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,64,0,1,fp8,fp8,0,0.49854934215545654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,64,128,1,float16,float16,0,0.16897600889205933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,64,128,1,float16,fp8,0,0.1654933293660482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,64,128,1,fp8,fp8,0,0.22402133544286093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,64,0,1,float16,float16,0,0.5124959945678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,64,128,1,float16,float16,0,0.17003200451533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,64,0,1,float16,fp8,0,0.5101546843846639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,64,128,1,float16,fp8,0,0.16909333070119223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,64,0,1,fp8,fp8,0,0.4831626812616984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,64,128,1,fp8,fp8,0,0.22107199827829996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,64,0,1,float16,float16,0,0.5217013359069824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,64,0,1,float16,fp8,0,0.52129598458608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,64,128,1,float16,float16,0,0.17325866222381592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,64,0,1,fp8,fp8,0,0.47619199752807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,64,128,1,float16,fp8,0,0.17255467176437378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,64,0,1,float16,float16,0,0.5141919851303101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,64,128,1,fp8,fp8,0,0.2290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,64,128,1,float16,float16,0,0.1275040010611216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,64,0,1,float16,fp8,0,0.5146239995956421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,64,0,1,fp8,fp8,0,0.478495995203654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,64,0,1,fp8,fp8,0,0.2781599958737691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,64,0,1,float16,float16,0,0.2977120081583659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,64,128,1,float16,fp8,0,0.12564266721407572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,64,128,1,float16,fp8,0,0.12242133418718974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,64,128,1,fp8,fp8,0,0.1595039963722229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,64,0,1,float16,fp8,0,0.29519466559092206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,64,128,1,float16,float16,0,0.12416533629099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,64,128,1,fp8,fp8,0,0.1546026666959127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,64,0,1,float16,float16,0,0.2937706708908081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,64,0,1,float16,fp8,0,0.29468266169230145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,64,128,1,float16,float16,0,0.12354666988054912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,64,0,1,fp8,fp8,0,0.27348800500233966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,64,128,1,float16,fp8,0,0.12267733613650005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,64,0,1,float16,float16,0,0.2967360019683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,64,128,1,fp8,fp8,0,0.15601600209871927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,64,0,1,float16,fp8,0,0.29604266087214154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,64,128,1,float16,float16,0,0.12372266252835591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,64,0,1,fp8,fp8,0,0.2747573256492615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,64,128,1,float16,fp8,0,0.12353066603342693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,64,0,1,float16,float16,0,0.29417065779368085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,64,128,1,fp8,fp8,0,0.15731733043988547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,64,0,1,float16,fp8,0,0.2985493342081706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,64,0,1,fp8,fp8,0,0.27556800842285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,64,128,1,float16,float16,0,0.8314346472422282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,64,128,1,float16,fp8,0,0.8197333017985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,64,128,1,fp8,fp8,0,1.068933327992757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,64,128,1,float16,float16,0,0.8783360322316488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,64,128,1,float16,fp8,0,0.8678186734517416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,64,0,1,float16,float16,0,2.3523200352986655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,64,0,1,fp8,fp8,0,2.186277389526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,64,0,1,float16,fp8,0,2.3844426472981772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,64,128,1,fp8,fp8,0,1.1091413497924805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,64,128,1,float16,float16,0,0.9018346468607584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,64,0,1,float16,float16,0,2.4773386319478354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,64,128,1,float16,fp8,0,0.8966399828592936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,64,0,1,float16,fp8,0,2.3894826571146646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,64,128,1,fp8,fp8,0,1.130511999130249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,64,128,1,float16,float16,0,0.49111465613047284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,64,0,1,fp8,fp8,0,2.207866668701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,64,128,1,float16,fp8,0,0.4834453264872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,64,0,1,float16,float16,0,2.439253330230713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,64,128,1,fp8,fp8,0,0.6216906706492106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,64,0,1,float16,float16,0,1.2822506427764893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,64,128,1,float16,float16,0,0.41708266735076904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,64,0,1,fp8,fp8,0,2.2287893295288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,64,0,1,float16,fp8,0,2.4240214029947915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,64,0,1,float16,fp8,0,1.2510986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,64,128,1,float16,fp8,0,0.41283198197682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,64,0,1,fp8,fp8,0,1.1894666353861492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,64,128,1,fp8,fp8,0,0.5448693434397379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,64,128,1,float16,float16,0,0.4207306702931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,64,0,1,float16,float16,0,1.1964960098266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,64,128,1,float16,fp8,0,0.42237865924835205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,64,0,1,float16,fp8,0,1.1844106515248616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,64,0,1,fp8,fp8,0,1.0960746606190999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,64,128,1,fp8,fp8,0,0.5617173512776693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,64,0,1,float16,float16,0,1.2232320308685303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,64,128,1,float16,float16,0,0.44069866339365643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,64,128,1,float16,fp8,0,0.42848531405131024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,64,0,1,float16,fp8,0,1.2011360327402751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,64,0,1,fp8,fp8,0,1.1443999608357747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,64,128,1,fp8,fp8,0,0.5791039864222208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,64,128,1,float16,float16,0,0.24961066246032715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,64,0,1,float16,float16,0,1.2298613389333088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,64,128,1,float16,fp8,0,0.2526666720708211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,64,128,1,fp8,fp8,0,0.3253119985262553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,64,0,1,float16,fp8,0,1.2277759710947673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,64,0,1,float16,float16,0,0.6555946667989095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,64,0,1,fp8,fp8,0,1.1449546813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,64,128,1,float16,float16,0,0.22878400484720865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,64,0,1,float16,fp8,0,0.6473439931869507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,64,0,1,fp8,fp8,0,0.5848480065663656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,64,128,1,float16,fp8,0,0.22526933749516806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,64,128,1,fp8,fp8,0,0.29763732353846234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,64,128,1,float16,float16,0,0.23201600710550943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,64,0,1,float16,float16,0,0.6338773171106974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,64,128,1,float16,fp8,0,0.22709866364796957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,64,0,1,float16,fp8,0,0.6313440004984537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,64,0,1,fp8,fp8,0,0.5490933259328207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,64,0,1,float16,float16,0,0.6365013519922892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,64,128,1,fp8,fp8,0,0.30425065755844116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,64,128,1,float16,float16,0,0.23355199893315634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,64,0,1,float16,fp8,0,0.633407990137736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,64,0,1,fp8,fp8,0,0.553274671236674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,64,128,1,float16,fp8,0,0.23135999838511148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,64,0,1,float16,float16,0,0.635317325592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,64,128,1,fp8,fp8,0,0.3044266700744629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,64,128,1,float16,float16,0,0.14421332875887552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,64,0,1,float16,fp8,0,0.6394773324330648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,64,0,1,fp8,fp8,0,0.567845344543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,64,0,1,float16,float16,0,0.327290674050649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,64,0,1,float16,fp8,0,0.32548266649246216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,64,128,1,float16,fp8,0,0.14433599511782327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,64,128,1,fp8,fp8,0,0.19012266397476196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,64,128,1,float16,float16,0,0.13223999738693237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,64,0,1,fp8,fp8,0,0.3059680064519246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,64,128,1,float16,fp8,0,0.1300320029258728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,64,0,1,float16,fp8,0,0.3128426671028137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,64,0,1,float16,float16,0,0.3185013333956401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,64,128,1,fp8,fp8,0,0.1747786601384481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,64,0,1,fp8,fp8,0,0.2933066685994466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,64,128,1,float16,float16,0,0.13272533814112344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,64,0,1,float16,float16,0,0.31881600618362427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,64,128,1,float16,fp8,0,0.13038933277130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,64,128,1,fp8,fp8,0,0.1750239928563436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,64,0,1,float16,fp8,0,0.31455467144648236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,64,128,1,float16,float16,0,0.1345919966697693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,64,0,1,fp8,fp8,0,0.29764266808827716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,64,128,1,float16,fp8,0,0.13344533244768778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,64,0,1,float16,float16,0,0.3180053234100342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,64,128,1,fp8,fp8,0,0.18050134181976318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,64,0,1,float16,fp8,0,0.31665066878000897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,64,128,1,float16,float16,0,0.1011946698029836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,64,0,1,fp8,fp8,0,0.2977493405342102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,64,128,1,float16,fp8,0,0.10109333197275798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,64,0,1,float16,float16,0,0.1919893423716227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,64,128,1,fp8,fp8,0,0.12812266747156778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,64,0,1,float16,fp8,0,0.19104532400767008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,64,0,1,fp8,fp8,0,0.17855999867121378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,64,128,1,fp8,fp8,0,0.11544000109036763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,64,128,1,float16,float16,0,0.09732266267140706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,64,0,1,float16,float16,0,0.18839999039967856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,64,128,1,float16,fp8,0,0.0974666674931844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,64,0,1,float16,fp8,0,0.19003732999165854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,64,0,1,fp8,fp8,0,0.17580799261728922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,64,128,1,float16,float16,0,0.09717333316802979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,64,128,1,float16,fp8,0,0.0971999963124593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,64,0,1,float16,fp8,0,0.1902773380279541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,64,0,1,float16,float16,0,0.18994667132695517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,64,128,1,fp8,fp8,0,0.11608533064524333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,64,0,1,fp8,fp8,0,0.1771679917971293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,64,128,1,float16,float16,0,0.09897067149480183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,64,0,1,float16,float16,0,0.19064533710479736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,64,128,1,float16,fp8,0,0.09727467099825542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,64,128,1,fp8,fp8,0,0.11896000305811565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,64,0,1,float16,fp8,0,0.19153600931167603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,64,0,1,fp8,fp8,0,0.17783466974894205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,64,128,1,float16,float16,0,1.105552037556966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,64,128,1,float16,fp8,0,1.0983359813690186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,64,128,1,fp8,fp8,0,1.404688040415446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,64,128,1,float16,float16,0,1.1574506759643555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,64,0,1,float16,float16,0,2.3458827336629233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,64,0,1,fp8,fp8,0,2.168922742207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,64,0,1,float16,fp8,0,2.329648017883301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,64,128,1,float16,fp8,0,1.1468853155771892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,64,128,1,fp8,fp8,0,1.4509387016296387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,64,0,1,float16,float16,0,2.3886186281840005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,64,128,1,float16,float16,0,1.191333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,64,0,1,float16,fp8,0,2.381253401438395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,64,0,1,fp8,fp8,0,2.1996533075968423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,64,128,1,float16,fp8,0,1.1728426615397136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,64,128,1,fp8,fp8,0,1.4755733807881672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,64,128,1,float16,float16,0,0.6526293357213339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,64,128,1,float16,fp8,0,0.6369386514027914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,64,0,1,float16,float16,0,2.4230186144510903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,64,0,1,float16,float16,0,1.2813493410746257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,64,0,1,float16,fp8,0,2.403674602508545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,64,128,1,fp8,fp8,0,0.8124799728393555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,64,0,1,fp8,fp8,0,2.230021317799886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,64,128,1,float16,float16,0,0.5429333448410034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,64,0,1,float16,fp8,0,1.2602240244547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,64,0,1,fp8,fp8,0,1.1890453497568767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,64,128,1,float16,fp8,0,0.5466080109278361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,64,0,1,float16,float16,0,1.166698694229126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,64,128,1,fp8,fp8,0,0.7117439905802408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,64,128,1,float16,float16,0,0.5807786782582601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,64,0,1,float16,fp8,0,1.1588106950124104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,64,0,1,fp8,fp8,0,1.076149304707845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,64,128,1,float16,fp8,0,0.5725813309351603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,64,128,1,fp8,fp8,0,0.7560266653696696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,64,0,1,float16,float16,0,1.2234186331431072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,64,128,1,float16,float16,0,0.5954666535059611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,64,128,1,float16,fp8,0,0.6093279918034872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,64,0,1,fp8,fp8,0,1.1146506468454997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,64,0,1,float16,fp8,0,1.2178080081939697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,64,128,1,fp8,fp8,0,0.7535839875539144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,64,128,1,float16,float16,0,0.3274186650911967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,64,0,1,float16,float16,0,1.2169493039449055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,64,128,1,float16,fp8,0,0.3161226709683736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,64,0,1,float16,fp8,0,1.2337600390116374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,64,0,1,float16,float16,0,0.6406186819076538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,64,0,1,fp8,fp8,0,1.1550400257110596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,64,128,1,fp8,fp8,0,0.4216959873835246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,64,128,1,float16,float16,0,0.2800106604894002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,64,0,1,float16,fp8,0,0.6367946863174438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,64,0,1,float16,float16,0,0.598906675974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,64,0,1,fp8,fp8,0,0.5858506759007772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,64,128,1,float16,fp8,0,0.282858669757843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,64,128,1,fp8,fp8,0,0.371829350789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,64,0,1,float16,fp8,0,0.6124000151952108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,64,128,1,float16,float16,0,0.2897119919459025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,64,0,1,fp8,fp8,0,0.5480106671651205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,64,128,1,float16,fp8,0,0.27938133478164673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,64,0,1,float16,float16,0,0.6058080196380615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,64,128,1,fp8,fp8,0,0.3764959971110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,64,128,1,float16,float16,0,0.29736532767613727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,64,0,1,float16,fp8,0,0.6177599827448527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,64,128,1,float16,fp8,0,0.2957493265469869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,64,0,1,fp8,fp8,0,0.5490826765696207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,64,0,1,float16,float16,0,0.6199306646982828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,64,128,1,fp8,fp8,0,0.377072016398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,64,128,1,float16,float16,0,0.17697600523630777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,64,0,1,float16,fp8,0,0.6194026470184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,64,0,1,fp8,fp8,0,0.5606133143107096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,64,128,1,float16,fp8,0,0.17474132776260376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,64,0,1,float16,float16,0,0.3290666739145915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,64,128,1,fp8,fp8,0,0.22849067052205405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,64,0,1,float16,fp8,0,0.32586665948232013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,64,128,1,float16,float16,0,0.15849600235621134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,64,0,1,fp8,fp8,0,0.298634668191274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,64,0,1,float16,float16,0,0.3076266646385193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,64,128,1,float16,fp8,0,0.15652267138163248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,64,128,1,fp8,fp8,0,0.2080693244934082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,64,0,1,float16,fp8,0,0.30763200918833417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,64,0,1,fp8,fp8,0,0.28752533594767254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,64,128,1,float16,float16,0,0.16004799803098044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,64,0,1,float16,float16,0,0.307370662689209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,64,128,1,float16,fp8,0,0.15763200322786966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,64,128,1,fp8,fp8,0,0.21370667219161987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,64,0,1,float16,fp8,0,0.3038133382797241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,64,128,1,float16,float16,0,0.16170133153597513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,64,0,1,fp8,fp8,0,0.2924213409423828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,64,0,1,float16,float16,0,0.3078346649805705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,64,128,1,float16,fp8,0,0.16116799910863241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,64,128,1,fp8,fp8,0,0.21748799085617065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,64,0,1,float16,fp8,0,0.30766934156417847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,64,128,1,float16,float16,0,0.10155199964841206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,64,0,1,fp8,fp8,0,0.2909333308537801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,64,0,1,float16,float16,0,0.17753599087397257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,64,128,1,float16,fp8,0,0.10342933734258015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,64,128,1,fp8,fp8,0,0.13798933227856955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,64,0,1,float16,fp8,0,0.17736534277598062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,64,0,1,fp8,fp8,0,0.16774400075276694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,64,128,1,float16,float16,0,0.09266133109728496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,64,0,1,float16,float16,0,0.1657813290754954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,64,128,1,float16,fp8,0,0.0944640040397644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,64,128,1,fp8,fp8,0,0.11541866262753804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,64,0,1,float16,fp8,0,0.16589867075284323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,64,0,1,fp8,fp8,0,0.15833066900571188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,64,128,1,fp8,fp8,0,0.1269866625467936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,64,128,1,float16,float16,0,0.09444800019264221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,64,0,1,float16,float16,0,0.16779200236002603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,64,128,1,float16,fp8,0,0.09468799829483032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,64,0,1,float16,fp8,0,0.1664906640847524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,64,0,1,fp8,fp8,0,0.15801599621772766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,64,128,1,float16,float16,0,0.09673600395520528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,64,0,1,float16,float16,0,0.1705066760381063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,64,128,1,float16,fp8,0,0.09637332955996196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,64,128,1,fp8,fp8,0,0.1293280025323232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,64,0,1,float16,fp8,0,0.16900799671808878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,64,0,1,fp8,fp8,0,0.16003732879956564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,64,128,1,float16,float16,0,0.07207466661930084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,64,0,1,float16,float16,0,0.11075199643770854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,64,128,1,float16,fp8,0,0.07250133156776428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,64,128,1,fp8,fp8,0,0.09511466821034749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,64,0,1,float16,fp8,0,0.1097813347975413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,64,0,1,fp8,fp8,0,0.10449600219726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,64,128,1,float16,float16,0,0.0697866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,64,0,1,float16,float16,0,0.1081119974454244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,64,128,1,float16,fp8,0,0.06993600229422252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,64,128,1,fp8,fp8,0,0.08616000413894653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,64,0,1,float16,fp8,0,0.10873599847157796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,64,0,1,fp8,fp8,0,0.10213333368301392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,64,128,1,float16,float16,0,0.06963733335336049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,64,0,1,float16,float16,0,0.1081119974454244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,64,128,1,float16,fp8,0,0.06960000097751617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,64,128,1,fp8,fp8,0,0.08541867136955261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,64,0,1,float16,fp8,0,0.10822932918866475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,64,0,1,fp8,fp8,0,0.10243733723958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,64,128,1,float16,float16,0,0.06958400209744771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,64,0,1,float16,float16,0,0.1083626647790273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,64,128,1,float16,fp8,0,0.06969066460927327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,64,128,1,fp8,fp8,0,0.08639466762542725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,64,0,1,float16,fp8,0,0.10845866799354553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,64,0,1,fp8,fp8,0,0.10198932886123657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,64,128,1,float16,float16,0,0.8174986839294434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,64,128,1,float16,fp8,0,0.8121013641357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,64,128,1,fp8,fp8,0,1.0322879950205486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,64,0,1,float16,float16,0,1.4545173645019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,64,128,1,float16,float16,0,0.8438933690388998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,64,0,1,fp8,fp8,0,1.3328959941864014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,64,0,1,float16,fp8,0,1.444661299387614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,64,128,1,float16,fp8,0,0.8374026616414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,64,0,1,float16,float16,0,1.4820213317871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,64,128,1,fp8,fp8,0,1.0766080220540364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,64,128,1,float16,float16,0,0.8708053429921468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,64,0,1,float16,fp8,0,1.4752693176269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,64,128,1,float16,fp8,0,0.8597013155619303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,64,0,1,fp8,fp8,0,1.3652052879333496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,64,128,1,fp8,fp8,0,1.0966506799062092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,64,0,1,float16,float16,0,1.5119199752807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,64,128,1,float16,float16,0,0.485370675722758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,64,128,1,float16,fp8,0,0.47534934679667157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,64,0,1,float16,fp8,0,1.5016320546468098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,64,0,1,float16,float16,0,0.8149173259735107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,64,0,1,fp8,fp8,0,1.3949492772420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,64,128,1,fp8,fp8,0,0.6071039835611979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,64,128,1,float16,float16,0,0.40673065185546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,64,0,1,fp8,fp8,0,0.7531733512878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,64,0,1,float16,fp8,0,0.8010186354319254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,64,128,1,float16,fp8,0,0.39977598190307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,64,0,1,float16,float16,0,0.7330773671468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,64,128,1,fp8,fp8,0,0.5333120028177897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,64,128,1,float16,float16,0,0.41743465264638263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,64,0,1,float16,fp8,0,0.7234453360239664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,64,0,1,fp8,fp8,0,0.6835573514302572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,64,128,1,float16,fp8,0,0.4127360184987386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,64,128,1,fp8,fp8,0,0.5439466635386149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,64,0,1,float16,float16,0,0.7395733197530111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,64,0,1,float16,fp8,0,0.7363519668579102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,64,128,1,float16,float16,0,0.43055466810862225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,64,0,1,fp8,fp8,0,0.6989493370056152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,64,128,1,float16,fp8,0,0.42819201946258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,64,0,1,float16,float16,0,0.7538879712422689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,64,128,1,fp8,fp8,0,0.5603839953740438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,64,128,1,float16,float16,0,0.24661332368850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,64,0,1,float16,fp8,0,0.746666669845581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,64,0,1,fp8,fp8,0,0.7153653303782145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,64,128,1,float16,fp8,0,0.24405866861343384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,64,128,1,fp8,fp8,0,0.31594665845235187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,64,0,1,float16,float16,0,0.4170773426691691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,64,0,1,float16,fp8,0,0.41442668437957764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,64,128,1,float16,float16,0,0.2146186629931132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,64,0,1,fp8,fp8,0,0.36855467160542804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,64,128,1,float16,fp8,0,0.2148853341738383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,64,0,1,float16,float16,0,0.38921598593393963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,64,128,1,fp8,fp8,0,0.28736533721288043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,64,128,1,float16,float16,0,0.21845867236455283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,64,0,1,fp8,fp8,0,0.34616533915201825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,64,0,1,float16,fp8,0,0.38701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,64,0,1,float16,float16,0,0.38758401075998944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,64,128,1,float16,fp8,0,0.21861867109934488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,64,128,1,fp8,fp8,0,0.29256532589594525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,64,0,1,float16,fp8,0,0.38885335127512616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,64,0,1,fp8,fp8,0,0.34462932745615643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,64,128,1,float16,float16,0,0.22686932484308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,64,128,1,float16,fp8,0,0.22101332743962607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,64,0,1,float16,float16,0,0.39763200283050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,64,128,1,fp8,fp8,0,0.2932906746864319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,64,0,1,float16,fp8,0,0.3951253493626912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,64,128,1,float16,float16,0,0.1357493301232656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,64,0,1,fp8,fp8,0,0.3545386791229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,64,0,1,float16,float16,0,0.21353065967559814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,64,128,1,float16,fp8,0,0.1365386644999186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,64,0,1,float16,fp8,0,0.20990933974583945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,64,128,1,fp8,fp8,0,0.18097599347432455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,64,0,1,fp8,fp8,0,0.20097599426905313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,64,128,1,float16,float16,0,0.12109333276748657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,64,0,1,float16,float16,0,0.1970613400141398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,64,128,1,float16,fp8,0,0.12204266587893169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,64,128,1,fp8,fp8,0,0.16646400094032288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,64,0,1,float16,fp8,0,0.1937546730041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,64,128,1,float16,float16,0,0.12435199817021687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,64,0,1,fp8,fp8,0,0.1845866640408834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,64,0,1,float16,fp8,0,0.1968266765276591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,64,0,1,float16,float16,0,0.19562667608261108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,64,128,1,float16,fp8,0,0.12277866403261821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,64,128,1,fp8,fp8,0,0.16736533244450888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,64,0,1,fp8,fp8,0,0.1854133407274882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,64,128,1,float16,float16,0,0.1267573336760203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,64,128,1,float16,fp8,0,0.12644267082214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,64,0,1,float16,float16,0,0.19977599382400513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,64,128,1,fp8,fp8,0,0.17007466157277426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,64,0,1,float16,fp8,0,0.20067733526229858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,64,0,1,fp8,fp8,0,0.19169066349665323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,64,128,1,float16,float16,0,0.08072000245253245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,64,0,1,float16,float16,0,0.11716799934705098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,64,128,1,float16,fp8,0,0.08290666838486989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,64,128,1,fp8,fp8,0,0.11121599872907002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,64,0,1,float16,fp8,0,0.11826133728027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,64,0,1,fp8,fp8,0,0.1148373285929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,64,128,1,float16,float16,0,0.07539733250935872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,64,0,1,float16,float16,0,0.11044266819953918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,64,128,1,float16,fp8,0,0.07499733567237854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,64,128,1,fp8,fp8,0,0.0939359962940216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,64,0,1,float16,fp8,0,0.11165866255760193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,64,0,1,fp8,fp8,0,0.10388799508412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,64,128,1,float16,float16,0,0.07460266848405202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,64,0,1,float16,float16,0,0.11226666967074077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,64,128,1,float16,fp8,0,0.07587199906508128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,64,128,1,fp8,fp8,0,0.09367466966311137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,64,0,1,float16,fp8,0,0.11269866426785786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,64,0,1,fp8,fp8,0,0.10538666447003682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,64,128,1,float16,float16,0,0.0775679995616277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,64,0,1,float16,float16,0,0.11209066708882649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,64,128,1,float16,fp8,0,0.07685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,64,128,1,fp8,fp8,0,0.0958186686038971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,64,0,1,float16,fp8,0,0.11276800433794658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,64,0,1,fp8,fp8,0,0.10678933064142863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,64,128,1,float16,float16,0,0.05794133245944977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,64,0,1,float16,float16,0,0.07764799892902374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,64,128,1,float16,fp8,0,0.05739733576774597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,64,128,1,fp8,fp8,0,0.06828799843788147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,64,0,1,float16,fp8,0,0.0767626663049062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,64,0,1,fp8,fp8,0,0.07315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,64,128,1,float16,float16,0,0.05699199934800466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,64,0,1,float16,float16,0,0.07688533266385396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,64,128,1,float16,fp8,0,0.057855998476346336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,64,128,1,fp8,fp8,0,0.06710933148860931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,64,0,1,float16,fp8,0,0.07611733178297679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,64,0,1,fp8,fp8,0,0.07232533395290375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,64,128,1,float16,float16,0,0.057002668579419456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,64,0,1,float16,float16,0,0.07693866888682048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,64,128,1,float16,fp8,0,0.05710933109124502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,64,128,1,fp8,fp8,0,0.06676266590754192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,64,0,1,float16,fp8,0,0.07769066592057546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,64,0,1,fp8,fp8,0,0.07203199962774913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,64,128,1,float16,float16,0,0.05686399837334951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,64,0,1,float16,float16,0,0.07729066908359528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,64,128,1,float16,fp8,0,0.05717866619427999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,64,128,1,fp8,fp8,0,0.06724266707897186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,64,0,1,float16,fp8,0,0.07748266557852428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,64,0,1,fp8,fp8,0,0.07229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,64,128,1,float16,float16,0,1.0805866718292236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,64,128,1,float16,fp8,0,1.0629173119862874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,64,128,1,fp8,fp8,0,1.334181308746338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,64,0,1,float16,float16,0,1.5415253639221191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,64,0,1,float16,fp8,0,1.5332694053649902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,64,128,1,float16,float16,0,1.0924159685770671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,64,0,1,fp8,fp8,0,1.3897919654846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,64,128,1,float16,fp8,0,1.0726772944132488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,64,0,1,float16,float16,0,1.5589332580566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,64,128,1,fp8,fp8,0,1.3549013137817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,64,128,1,float16,float16,0,1.1293013095855713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,64,0,1,float16,fp8,0,1.5368053118387859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,64,0,1,fp8,fp8,0,1.4105547269185383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,64,0,1,float16,float16,0,1.5985120137532551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,64,128,1,float16,fp8,0,1.1073386669158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,64,128,1,float16,float16,0,0.6310933430989584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,64,128,1,fp8,fp8,0,1.4224692980448406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,64,0,1,float16,float16,0,0.8696373303731283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,64,128,1,float16,fp8,0,0.6195573409398397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,64,0,1,fp8,fp8,0,1.469136079152425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,64,0,1,float16,fp8,0,1.574453353881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,64,128,1,fp8,fp8,0,0.7830186684926351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,64,0,1,float16,fp8,0,0.8511040210723877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,64,0,1,fp8,fp8,0,0.8049066861470541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,64,128,1,float16,float16,0,0.5349973440170288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,64,128,1,float16,fp8,0,0.525328000386556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,64,0,1,float16,float16,0,0.7818453311920166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,64,128,1,fp8,fp8,0,0.6814826329549154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,64,0,1,float16,fp8,0,0.7610507011413574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,64,128,1,float16,float16,0,0.551578680674235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,64,0,1,fp8,fp8,0,0.7016692956288656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,64,0,1,float16,float16,0,0.7828906377156576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,64,128,1,float16,fp8,0,0.5410186847050985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,64,128,1,fp8,fp8,0,0.7038933436075846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,64,0,1,float16,fp8,0,0.7828213373819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,64,0,1,fp8,fp8,0,0.7145546277364095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,64,128,1,float16,float16,0,0.572703997294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,64,128,1,float16,fp8,0,0.5646933317184448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,64,0,1,float16,float16,0,0.8080586592356364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,64,128,1,fp8,fp8,0,0.7131840387980143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,64,128,1,float16,float16,0,0.3264906605084737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,64,0,1,float16,fp8,0,0.8009173075358073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,64,0,1,float16,float16,0,0.4467200040817261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,64,0,1,fp8,fp8,0,0.7443946997324625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,64,128,1,float16,fp8,0,0.3189919988314311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,64,128,1,fp8,fp8,0,0.40612268447875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,64,0,1,float16,fp8,0,0.43936534722646076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,64,0,1,float16,float16,0,0.40493865807851154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,64,128,1,float16,float16,0,0.2791893283526103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,64,0,1,fp8,fp8,0,0.40784533818562824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,64,128,1,float16,fp8,0,0.2733760078748067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,64,128,1,fp8,fp8,0,0.3641066551208496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,64,0,1,float16,fp8,0,0.39693331718444824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,64,128,1,float16,float16,0,0.28275734186172485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,64,0,1,fp8,fp8,0,0.36268798510233563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,64,0,1,float16,float16,0,0.4070133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,64,128,1,float16,fp8,0,0.27982399861017865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,64,128,1,fp8,fp8,0,0.36929599444071454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,64,128,1,float16,float16,0,0.2897653381029765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,64,0,1,float16,fp8,0,0.3982400099436442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,64,0,1,fp8,fp8,0,0.3635679880777995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,64,128,1,float16,fp8,0,0.28498133023579914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,64,0,1,float16,float16,0,0.4119199911753337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,64,128,1,fp8,fp8,0,0.3803146680196126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,64,128,1,float16,float16,0,0.17209599415461221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,64,0,1,float16,fp8,0,0.40998931725819904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,64,0,1,float16,fp8,0,0.23227200905481973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,64,0,1,fp8,fp8,0,0.3765439987182617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,64,0,1,float16,float16,0,0.23519466320673624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,64,128,1,float16,fp8,0,0.16917866468429565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,64,128,1,fp8,fp8,0,0.22028799851735434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,64,0,1,fp8,fp8,0,0.21222400665283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,64,0,1,float16,fp8,0,0.20107199748357138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,64,128,1,float16,float16,0,0.15039466818173727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,64,0,1,float16,float16,0,0.20138132572174072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,64,128,1,float16,fp8,0,0.14758933583895364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,64,128,1,fp8,fp8,0,0.20221332708994547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,64,128,1,fp8,fp8,0,0.20409067471822104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,64,0,1,fp8,fp8,0,0.19142399231592813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,64,128,1,float16,float16,0,0.15243732929229736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,64,0,1,float16,float16,0,0.206661323706309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,64,128,1,float16,fp8,0,0.15169599652290344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,64,0,1,float16,fp8,0,0.20168532927831015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,64,0,1,fp8,fp8,0,0.19464532534281412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,64,128,1,float16,float16,0,0.15625066558519998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,64,0,1,float16,float16,0,0.20689600706100464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,64,128,1,float16,fp8,0,0.15602133671442667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,64,128,1,fp8,fp8,0,0.20845866203308105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,64,0,1,float16,fp8,0,0.20964266856511435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,64,128,1,fp8,fp8,0,0.1301866670449575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,64,0,1,fp8,fp8,0,0.20031466086705527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,64,128,1,float16,float16,0,0.09742933511734009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,64,0,1,float16,float16,0,0.12120532989501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,64,128,1,float16,fp8,0,0.09727467099825542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,64,0,1,float16,fp8,0,0.12160533666610718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,64,0,1,fp8,fp8,0,0.11706667145093282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,64,0,1,fp8,fp8,0,0.10358933607737224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,64,128,1,float16,float16,0,0.08738666772842407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,64,0,1,float16,float16,0,0.1113866666952769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,64,128,1,float16,fp8,0,0.08710933725039165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,64,128,1,fp8,fp8,0,0.11017066240310669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,64,0,1,float16,fp8,0,0.10947733124097188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,64,128,1,float16,float16,0,0.08874666690826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,64,0,1,float16,float16,0,0.11135466893513997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,64,128,1,float16,fp8,0,0.08667733271916707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,64,128,1,fp8,fp8,0,0.12011200189590454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,64,0,1,float16,fp8,0,0.1125866671403249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,64,0,1,fp8,fp8,0,0.10648533701896667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,64,128,1,float16,float16,0,0.08994133273760478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,64,0,1,float16,float16,0,0.11410666505495708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,64,128,1,float16,fp8,0,0.08902933200200398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,64,128,1,fp8,fp8,0,0.12286933263142903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,64,0,1,float16,fp8,0,0.11345600088437398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,64,0,1,fp8,fp8,0,0.11150399843851726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,64,128,1,float16,float16,0,0.05952000121275584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,64,0,1,float16,float16,0,0.06993600229422252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,64,128,1,float16,fp8,0,0.05843733251094818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,64,128,1,fp8,fp8,0,0.08014399806658427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,64,0,1,float16,fp8,0,0.07073600093523662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,64,0,1,fp8,fp8,0,0.06924800078074138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,64,128,1,float16,float16,0,0.0554666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,64,0,1,float16,float16,0,0.06753600140412648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,64,128,1,float16,fp8,0,0.054901331663131714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,64,128,1,fp8,fp8,0,0.07228266696135204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,64,0,1,float16,fp8,0,0.06723733246326447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,64,0,1,fp8,fp8,0,0.06361599763234456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,64,128,1,float16,float16,0,0.05471999943256378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,64,0,1,float16,float16,0,0.06899199883143108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,64,128,1,float16,fp8,0,0.05532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,64,128,1,fp8,fp8,0,0.07242133220036824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,64,0,1,float16,fp8,0,0.06764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,64,0,1,fp8,fp8,0,0.06342933575312297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,64,128,1,float16,float16,0,0.05532266696294149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,64,0,1,float16,float16,0,0.06853333115577698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,64,128,1,float16,fp8,0,0.05665066838264465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,64,128,1,fp8,fp8,0,0.07289066910743713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,64,0,1,float16,fp8,0,0.06808533271153767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,64,128,1,fp8,fp8,0,0.050293331344922386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,64,0,1,fp8,fp8,0,0.0652159998814265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,64,128,1,float16,float16,0,0.04498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,64,0,1,float16,float16,0,0.04933333396911621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,64,128,1,float16,fp8,0,0.046469335754712425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,64,0,1,float16,fp8,0,0.0490880012512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,64,0,1,fp8,fp8,0,0.04747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,64,128,1,float16,float16,0,0.04709866642951965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,64,0,1,float16,float16,0,0.04829333225886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,64,128,1,float16,fp8,0,0.04446400205294291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,64,128,1,fp8,fp8,0,0.04983466863632202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,64,0,1,float16,fp8,0,0.048826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,64,0,1,fp8,fp8,0,0.04737600187460581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,64,128,1,float16,float16,0,0.04789333542188009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,64,0,1,float16,float16,0,0.04886933167775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,64,128,1,float16,fp8,0,0.04797866443792979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,64,128,1,fp8,fp8,0,0.049925332268079124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,64,0,1,float16,fp8,0,0.048810665806134544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,64,0,1,fp8,fp8,0,0.048101335763931274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,64,128,1,float16,float16,0,0.04837866624196371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,64,0,1,float16,float16,0,0.048570667703946434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,64,128,1,float16,fp8,0,0.048021331429481506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,64,128,1,fp8,fp8,0,0.05040533343950907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,64,0,1,float16,fp8,0,0.04870399832725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,64,0,1,fp8,fp8,0,0.04757866760094961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,64,128,1,float16,float16,0,0.8294666608174642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,64,128,1,float16,fp8,0,0.8196960290273031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,64,0,1,float16,float16,0,1.0423413117726643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,64,128,1,fp8,fp8,0,1.0201706886291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,64,0,1,float16,fp8,0,1.03548796971639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,64,0,1,fp8,fp8,0,0.9307733376820883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,64,128,1,float16,float16,0,0.8331627051035563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,64,0,1,float16,float16,0,1.0497173468271892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,64,128,1,float16,fp8,0,0.8219839731852213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,64,128,1,fp8,fp8,0,1.0357066790262859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,64,0,1,float16,fp8,0,1.032650629679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,64,0,1,fp8,fp8,0,0.9404906431833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,64,128,1,float16,float16,0,0.8617599805196127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,64,128,1,float16,fp8,0,0.8426613012949625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,64,0,1,float16,float16,0,1.0752533276875813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,64,128,1,float16,float16,0,0.4850826660792033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,64,128,1,fp8,fp8,0,1.0525333086649578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,64,0,1,float16,fp8,0,1.058351993560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,64,128,1,float16,fp8,0,0.47200000286102295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,64,0,1,float16,float16,0,0.5921973387400309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,64,0,1,fp8,fp8,0,0.9716479778289795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,64,128,1,fp8,fp8,0,0.5886293252309164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,64,0,1,float16,fp8,0,0.5855413277943929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,64,128,1,float16,float16,0,0.40437865257263184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,64,0,1,fp8,fp8,0,0.5385226806004842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,64,0,1,float16,float16,0,0.5188533465067545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,64,128,1,float16,fp8,0,0.40012800693511963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,64,128,1,fp8,fp8,0,0.5279680093129476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,64,0,1,float16,fp8,0,0.5112640062967936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,64,0,1,fp8,fp8,0,0.471504012743632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,64,0,1,float16,float16,0,0.52565864721934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,64,128,1,float16,float16,0,0.4237706661224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,64,128,1,fp8,fp8,0,0.5328746636708578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,64,128,1,float16,fp8,0,0.4166293144226074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,64,0,1,float16,fp8,0,0.5224800109863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,64,0,1,fp8,fp8,0,0.4886346658070882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,64,128,1,float16,float16,0,0.4320640166600545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,64,128,1,fp8,fp8,0,0.5507040023803711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,64,128,1,float16,fp8,0,0.4259200096130371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,64,0,1,float16,float16,0,0.5404320160547892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,64,0,1,float16,fp8,0,0.5403466622034708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,64,128,1,float16,float16,0,0.24952000379562378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,64,0,1,fp8,fp8,0,0.4980959892272949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,64,0,1,float16,float16,0,0.3052906592686971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,64,128,1,float16,fp8,0,0.24592000246047974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,64,128,1,fp8,fp8,0,0.3123306632041931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,64,0,1,float16,fp8,0,0.2993226647377014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,64,0,1,fp8,fp8,0,0.2775999903678894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,64,128,1,float16,float16,0,0.21199466784795126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,64,0,1,float16,float16,0,0.2704319953918457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,64,0,1,fp8,fp8,0,0.2422773241996765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,64,128,1,float16,fp8,0,0.20856000979741415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,64,128,1,fp8,fp8,0,0.2826186617215474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,64,0,1,float16,fp8,0,0.26766933997472125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,64,128,1,float16,float16,0,0.21852266788482666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,64,128,1,fp8,fp8,0,0.2848373254140218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,64,0,1,float16,float16,0,0.2721760074297587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,64,0,1,float16,fp8,0,0.2680533329645793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,64,128,1,float16,fp8,0,0.21317867437998453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,64,0,1,fp8,fp8,0,0.2500159939130147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,64,0,1,float16,float16,0,0.28108266989390057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,64,128,1,float16,float16,0,0.2244853377342224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,64,128,1,float16,fp8,0,0.2202933430671692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,64,128,1,fp8,fp8,0,0.2905493378639221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,64,0,1,float16,fp8,0,0.27639466524124146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,64,0,1,fp8,fp8,0,0.25286932786305744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,64,128,1,float16,float16,0,0.13577600320180258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,64,0,1,float16,fp8,0,0.1574133336544037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,64,0,1,float16,float16,0,0.1607306698958079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,64,128,1,float16,fp8,0,0.13401066263516745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,64,128,1,fp8,fp8,0,0.17332800229390463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,64,0,1,fp8,fp8,0,0.14618133505185446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,64,128,1,float16,float16,0,0.11741866668065389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,64,0,1,float16,float16,0,0.13845866918563843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,64,128,1,float16,fp8,0,0.1156160036722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,64,128,1,fp8,fp8,0,0.1602666676044464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,64,0,1,float16,fp8,0,0.1366986632347107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,64,0,1,fp8,fp8,0,0.130922665198644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,64,128,1,float16,float16,0,0.11814399560292561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,64,0,1,float16,float16,0,0.13894933462142944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,64,128,1,float16,fp8,0,0.11795199910799663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,64,128,1,fp8,fp8,0,0.1597813367843628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,64,0,1,float16,fp8,0,0.1404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,64,0,1,fp8,fp8,0,0.1341493328412374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,64,128,1,float16,float16,0,0.12140799562136333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,64,0,1,float16,float16,0,0.14356799920399985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,64,128,1,float16,fp8,0,0.12198399504025777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,64,128,1,fp8,fp8,0,0.16478400429089865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,64,0,1,float16,fp8,0,0.14128533005714417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,64,0,1,fp8,fp8,0,0.1395093301932017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,64,128,1,float16,float16,0,0.07714666426181793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,64,0,1,float16,float16,0,0.08508800466855367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,64,128,1,float16,fp8,0,0.07733866572380066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,64,128,1,fp8,fp8,0,0.10643200079600017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,64,0,1,float16,fp8,0,0.08700799942016602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,64,0,1,fp8,fp8,0,0.08526933193206787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,64,128,1,float16,float16,0,0.06748799979686737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,64,0,1,fp8,fp8,0,0.07448533177375793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,64,0,1,float16,float16,0,0.07867733140786488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,64,128,1,float16,fp8,0,0.06625600159168243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,64,128,1,fp8,fp8,0,0.08904533584912618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,64,0,1,float16,fp8,0,0.07772799829641978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,64,128,1,float16,float16,0,0.06835199892520905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,64,0,1,float16,float16,0,0.07937600215276082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,64,128,1,float16,fp8,0,0.06913066903750102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,64,128,1,fp8,fp8,0,0.08917333682378133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,64,0,1,float16,fp8,0,0.07754133145014445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,64,0,1,fp8,fp8,0,0.07589866717656453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,64,128,1,float16,float16,0,0.07097066442171733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,64,0,1,fp8,fp8,0,0.07725333174069722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,64,0,1,float16,float16,0,0.0800853321949641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,64,128,1,float16,fp8,0,0.04756266872088114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,64,128,1,float16,fp8,0,0.069882666071256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,64,128,1,fp8,fp8,0,0.09701333443323772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,64,0,1,float16,fp8,0,0.08022400240103404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,64,128,1,float16,float16,0,0.0469813346862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,64,0,1,float16,float16,0,0.053317333261171974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,64,128,1,float16,fp8,0,0.04587733248869578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,64,128,1,fp8,fp8,0,0.0611413319905599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,64,0,1,float16,fp8,0,0.052527998884518944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,64,0,1,fp8,fp8,0,0.0498986691236496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,64,128,1,float16,float16,0,0.04586133360862732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,64,0,1,float16,float16,0,0.05117333432038625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,64,128,1,fp8,fp8,0,0.05613866448402405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,64,0,1,fp8,fp8,0,0.04822400212287903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,64,0,1,float16,fp8,0,0.05159999926884969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,64,128,1,float16,float16,0,0.046256000796953835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,64,0,1,float16,float16,0,0.05166399975617727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,64,128,1,float16,fp8,0,0.046309332052866616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,64,128,1,fp8,fp8,0,0.056703999638557434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,64,0,1,float16,fp8,0,0.0516533354918162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,64,0,1,fp8,fp8,0,0.048394665122032166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,64,128,1,float16,float16,0,0.046442667643229164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,64,0,1,float16,float16,0,0.052149335543314614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,64,128,1,float16,fp8,0,0.04674133161703745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,64,128,1,fp8,fp8,0,0.0573226660490036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,64,0,1,float16,fp8,0,0.05166399975617727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,64,0,1,fp8,fp8,0,0.049269333481788635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,64,128,1,float16,float16,0,0.0403413325548172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,64,0,1,float16,float16,0,0.04048000027736028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,64,128,1,float16,fp8,0,0.04130133241415024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,64,128,1,fp8,fp8,0,0.04621333380540212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,64,0,1,float16,float16,0,0.04043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,64,0,1,float16,fp8,0,0.04077333211898804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,64,0,1,fp8,fp8,0,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,64,128,1,float16,float16,0,0.040965333580970764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,64,128,1,float16,fp8,0,0.039317332208156586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,64,128,1,fp8,fp8,0,0.04584533472855886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,64,0,1,float16,fp8,0,0.04108799993991852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,64,0,1,fp8,fp8,0,0.03995199998219808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,64,128,1,float16,float16,0,0.04014399896065394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,64,0,1,float16,float16,0,0.040661332507928215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,64,128,1,float16,float16,0,0.040448000033696495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,64,128,1,float16,fp8,0,0.04043200115362803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,64,128,1,fp8,fp8,0,0.045850664377212524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,64,128,1,fp8,fp8,0,0.04656533400217692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,64,0,1,float16,fp8,0,0.040336000422636666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,64,0,1,fp8,fp8,0,0.04029333343108495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,64,0,1,float16,float16,0,0.04035733391841253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,64,128,1,float16,fp8,0,0.04043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,64,0,1,float16,fp8,0,0.04083200047413508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,64,0,1,fp8,fp8,0,0.039781334499518074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,64,128,1,float16,float16,0,0.9394400119781494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,64,0,1,float16,float16,0,1.0488373438517253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,64,128,1,float16,fp8,0,0.9305493036905924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,64,128,1,fp8,fp8,0,1.218997319539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,64,0,1,float16,fp8,0,1.053066651026408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,64,0,1,fp8,fp8,0,0.9524373213450114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,64,128,1,float16,float16,0,0.9393493334452311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,64,0,1,float16,float16,0,1.0661706924438477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,64,128,1,float16,fp8,0,0.9369493325551351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,64,128,1,fp8,fp8,0,1.2560693422953289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,64,0,1,float16,fp8,0,1.0596319834391277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,64,0,1,fp8,fp8,0,0.9798186620076498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,64,128,1,float16,float16,0,0.9730400244394938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,64,0,1,float16,float16,0,1.0951840082804363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,64,128,1,float16,fp8,0,0.9763840039571127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,64,128,1,float16,float16,0,0.5645546515782675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,64,0,1,float16,float16,0,0.62336532274882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,64,128,1,float16,fp8,0,0.5485973358154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,64,128,1,fp8,fp8,0,1.2647573153177898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,64,0,1,fp8,fp8,0,1.0035040378570557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,64,0,1,float16,fp8,0,1.088037331899007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,64,128,1,fp8,fp8,0,0.6982080141703287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,64,0,1,float16,fp8,0,0.608725349108378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,64,0,1,fp8,fp8,0,0.5467946529388428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,64,128,1,float16,float16,0,0.47634132703145343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,64,0,1,float16,float16,0,0.5359040101369222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,64,128,1,float16,fp8,0,0.4726293484369914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,64,0,1,fp8,fp8,0,0.4872533480326335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,64,0,1,float16,fp8,0,0.5446720123291016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,64,128,1,fp8,fp8,0,0.6286933422088623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,64,128,1,float16,float16,0,0.48632001876831055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,64,0,1,float16,float16,0,0.5373546679814657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,64,128,1,float16,fp8,0,0.48096001148223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,64,128,1,fp8,fp8,0,0.6323413451512655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,64,0,1,float16,fp8,0,0.5388906796773275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,64,0,1,fp8,fp8,0,0.5027786493301392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,64,128,1,float16,float16,0,0.49008532365163165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,64,0,1,float16,float16,0,0.5610506534576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,64,128,1,float16,fp8,0,0.48637866973876953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,64,128,1,fp8,fp8,0,0.6548639933268229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,64,128,1,float16,float16,0,0.2930399974187215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,64,0,1,float16,fp8,0,0.5501066843668619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,64,0,1,float16,float16,0,0.3280373414357503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,64,0,1,fp8,fp8,0,0.515338659286499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,64,128,1,float16,fp8,0,0.28699199358622235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,64,128,1,fp8,fp8,0,0.36084266503651935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,64,0,1,float16,fp8,0,0.3173759977022807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,64,0,1,fp8,fp8,0,0.28938666979471844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,64,128,1,fp8,fp8,0,0.3372906843821208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,64,128,1,float16,float16,0,0.2503146727879842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,64,0,1,float16,float16,0,0.28466665744781494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,64,128,1,float16,float16,0,0.2557813326517741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,64,128,1,float16,fp8,0,0.25456533829371136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,64,0,1,float16,fp8,0,0.2826240062713623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,64,0,1,fp8,fp8,0,0.25734400749206543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,64,0,1,float16,float16,0,0.28937600056330365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,64,128,1,float16,fp8,0,0.25676266352335614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,64,128,1,fp8,fp8,0,0.3407306671142578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,64,0,1,float16,fp8,0,0.2861973245938619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,64,0,1,fp8,fp8,0,0.25867732365926105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,64,128,1,float16,float16,0,0.2641013264656067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,64,128,1,fp8,fp8,0,0.3431520064671834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,64,0,1,float16,float16,0,0.2906879981358846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,64,128,1,float16,fp8,0,0.26526399453481037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,64,0,1,float16,fp8,0,0.2953439950942993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,64,128,1,fp8,fp8,0,0.2000160018603007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,64,0,1,fp8,fp8,0,0.26099733511606854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,64,128,1,float16,float16,0,0.15896000464757284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,64,0,1,float16,float16,0,0.184714674949646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,64,128,1,float16,fp8,0,0.15677332878112793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,64,0,1,float16,fp8,0,0.1788533329963684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,64,0,1,fp8,fp8,0,0.1495253344376882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,64,128,1,float16,float16,0,0.13768532872200012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,64,0,1,float16,float16,0,0.15059733390808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,64,128,1,float16,fp8,0,0.13916800419489542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,64,128,1,fp8,fp8,0,0.18458133935928345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,64,0,1,float16,fp8,0,0.15094932913780212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,64,0,1,fp8,fp8,0,0.13667733470598856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,64,128,1,float16,float16,0,0.14220266540845236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,64,0,1,float16,float16,0,0.15217600266138712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,64,128,1,float16,fp8,0,0.14018133282661438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,64,128,1,fp8,fp8,0,0.18779732783635458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,64,0,1,float16,fp8,0,0.15059733390808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,64,0,1,fp8,fp8,0,0.13938666383425394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,64,128,1,float16,float16,0,0.14426666498184204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,64,0,1,float16,float16,0,0.15587733189264932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,64,128,1,float16,fp8,0,0.14560533563296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,64,128,1,fp8,fp8,0,0.1884373426437378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,64,0,1,float16,fp8,0,0.15290133158365884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,64,0,1,fp8,fp8,0,0.1411946713924408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,64,0,1,float16,fp8,0,0.0928106705347697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,64,128,1,float16,float16,0,0.09330133597056071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,64,0,1,float16,float16,0,0.09457600116729736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,64,0,1,float16,float16,0,0.08215466638406117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,64,128,1,float16,fp8,0,0.09151466687520345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,64,128,1,fp8,fp8,0,0.11926933129628499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,64,0,1,fp8,fp8,0,0.08573333422342937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,64,128,1,float16,float16,0,0.07803733150164287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,64,128,1,float16,fp8,0,0.07859200239181519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,64,128,1,fp8,fp8,0,0.10358933607737224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,64,0,1,float16,fp8,0,0.08272000153859456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,64,0,1,fp8,fp8,0,0.0765226682027181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,64,128,1,float16,float16,0,0.08012266457080841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,64,0,1,float16,float16,0,0.08434666196505229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,64,128,1,float16,fp8,0,0.07881066699822743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,64,128,1,fp8,fp8,0,0.10833066701889038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,64,0,1,float16,fp8,0,0.08385599652926128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,64,0,1,fp8,fp8,0,0.07739733159542084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,64,128,1,float16,float16,0,0.08447999755541484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,64,0,1,float16,float16,0,0.08852266271909077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,64,128,1,float16,fp8,0,0.08256533245245616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,64,128,1,fp8,fp8,0,0.1104213297367096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,64,0,1,float16,fp8,0,0.08726400136947632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,64,0,1,fp8,fp8,0,0.0798773318529129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,64,128,1,float16,float16,0,0.05301866432030996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,64,0,1,float16,float16,0,0.054885332783063255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,64,128,1,float16,fp8,0,0.05323199927806854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,64,128,1,fp8,fp8,0,0.07392000158627827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,64,0,1,float16,fp8,0,0.05381333331267039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,64,0,1,fp8,fp8,0,0.050474668542544045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,64,128,1,float16,float16,0,0.048309331138928734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,64,0,1,float16,float16,0,0.05042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,64,128,1,float16,fp8,0,0.04861866434415182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,64,128,1,fp8,fp8,0,0.06282133360703786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,64,0,1,float16,fp8,0,0.049914668003718056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,64,0,1,fp8,fp8,0,0.04577066500981649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,64,128,1,float16,float16,0,0.048613334695498146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,64,0,1,float16,float16,0,0.05064533154169718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,64,128,1,float16,fp8,0,0.048938666780789696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,64,128,1,fp8,fp8,0,0.06406400104363759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,64,0,1,float16,fp8,0,0.050767997900644936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,64,0,1,fp8,fp8,0,0.046853333711624146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,64,128,1,float16,float16,0,0.05006400247414907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,64,0,1,float16,float16,0,0.052000001072883606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,64,128,1,float16,fp8,0,0.05106666684150696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,64,128,1,fp8,fp8,0,0.0631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,64,128,1,fp8,fp8,0,0.042917331059773765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,64,0,1,fp8,fp8,0,0.048122664292653404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,64,0,1,float16,fp8,0,0.05227200190226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,64,128,1,float16,float16,0,0.03591466695070267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,64,128,1,float16,float16,0,0.040607998768488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,64,0,1,float16,float16,0,0.03733866661787033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,64,128,1,float16,fp8,0,0.04154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,64,0,1,float16,fp8,0,0.03455466777086258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,64,0,1,float16,fp8,0,0.038015998899936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,64,0,1,fp8,fp8,0,0.034101332227389015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,64,0,1,float16,float16,0,0.03501333296298981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,64,128,1,float16,fp8,0,0.03875733415285746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,64,128,1,fp8,fp8,0,0.04135466615358988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,64,0,1,fp8,fp8,0,0.0329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,64,128,1,float16,float16,0,0.037871999045213066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,64,0,1,float16,float16,0,0.03480000048875809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,64,128,1,float16,fp8,0,0.035829332967599235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,64,128,1,fp8,fp8,0,0.041482667128245033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,64,128,1,fp8,fp8,0,0.04163199911514918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,64,0,1,float16,fp8,0,0.03562133262554804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,64,0,1,fp8,fp8,0,0.03313600023587545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,64,128,1,float16,float16,0,0.03995733211437861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,64,0,1,float16,float16,0,0.0358240008354187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,64,128,1,float16,fp8,0,0.03668266783157984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,64,0,1,float16,fp8,0,0.03676266719897588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,64,0,1,fp8,fp8,0,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,64,128,1,float16,float16,0,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,64,0,1,float16,float16,0,0.029018667836983997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,64,128,1,float16,fp8,0,0.026367999613285065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,64,128,1,fp8,fp8,0,0.03283733377854029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,64,0,1,float16,fp8,0,0.028421332438786823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,64,0,1,fp8,fp8,0,0.027509334186712902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,64,128,1,float16,float16,0,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,64,0,1,float16,float16,0,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,64,128,1,float16,fp8,0,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,64,128,1,fp8,fp8,0,0.032399999598662056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,64,0,1,float16,fp8,0,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,64,0,1,fp8,fp8,0,0.02661866694688797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,64,128,1,float16,float16,0,0.024847999215126038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,64,0,1,float16,float16,0,0.02779199928045273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,64,128,1,float16,fp8,0,0.025807999074459076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,64,128,1,fp8,fp8,0,0.03173866619666418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,64,0,1,float16,fp8,0,0.027989332874615986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,64,0,1,fp8,fp8,0,0.026874666412671406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,64,128,1,float16,float16,0,0.025072000920772552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,64,0,1,float16,float16,0,0.027823999524116516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,64,128,1,float16,fp8,0,0.025674665967623394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,64,128,1,fp8,fp8,0,0.03229333211978277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,64,0,1,float16,fp8,0,0.027888000011444092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,64,0,1,fp8,fp8,0,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,64,128,1,float16,float16,0,0.8906613190968832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,64,0,1,float16,float16,0,0.8724319934844971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,64,128,1,float16,fp8,0,0.8771253426869711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,64,0,1,float16,fp8,0,0.8616800308227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,64,0,1,fp8,fp8,0,0.773418664932251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,64,128,1,fp8,fp8,0,1.1561546325683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,64,128,1,float16,float16,0,0.8906559944152832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,64,0,1,float16,float16,0,0.8705546855926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,64,128,1,float16,fp8,0,0.8972053527832031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,64,128,1,fp8,fp8,0,1.1657066345214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,64,0,1,float16,fp8,0,0.870410680770874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,64,0,1,fp8,fp8,0,0.7976000308990479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,64,128,1,float16,float16,0,0.9313759803771973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,64,0,1,float16,float16,0,0.9167199929555258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,64,128,1,float16,fp8,0,0.9293013413747152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,64,128,1,float16,float16,0,0.5404853423436483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,64,0,1,float16,float16,0,0.5265386501948038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,64,0,1,float16,fp8,0,0.9033439954121908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,64,128,1,float16,fp8,0,0.5244693358739217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,64,128,1,fp8,fp8,0,1.2067039807637532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,64,0,1,fp8,fp8,0,0.8325866858164469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,64,0,1,float16,fp8,0,0.5149279832839966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,64,128,1,fp8,fp8,0,0.6499626636505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,64,0,1,fp8,fp8,0,0.45033065478007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,64,0,1,float16,float16,0,0.4418506622314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,64,128,1,float16,float16,0,0.45532798767089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,64,128,1,float16,fp8,0,0.4514293273289998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,64,128,1,fp8,fp8,0,0.5929439862569174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,64,0,1,float16,fp8,0,0.44115734100341797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,64,0,1,fp8,fp8,0,0.4028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,64,128,1,float16,float16,0,0.45584531625111896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,64,0,1,float16,float16,0,0.4455466667811076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,64,128,1,float16,fp8,0,0.46346668402353924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,64,0,1,float16,fp8,0,0.44554134209950763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,64,128,1,fp8,fp8,0,0.5957546631495158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,64,0,1,fp8,fp8,0,0.4005013306935628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,64,128,1,float16,float16,0,0.4742186864217122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,64,0,1,float16,float16,0,0.45519999663035077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,64,128,1,float16,fp8,0,0.4651999870936076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,64,128,1,float16,float16,0,0.28353599707285565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,64,128,1,fp8,fp8,0,0.6249866485595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,64,0,1,fp8,fp8,0,0.41549865404764813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,64,0,1,float16,fp8,0,0.4557493527730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,64,128,1,float16,fp8,0,0.2770506739616394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,64,0,1,float16,float16,0,0.27504533529281616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,64,128,1,fp8,fp8,0,0.3396586577097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,64,0,1,float16,fp8,0,0.26893333594004315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,64,0,1,fp8,fp8,0,0.23029865821202597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,64,128,1,float16,float16,0,0.24035733938217163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,64,0,1,float16,float16,0,0.23331733544667563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,64,0,1,fp8,fp8,0,0.20756800969441733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,64,128,1,float16,fp8,0,0.2399466633796692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,64,128,1,fp8,fp8,0,0.31734933455785114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,64,0,1,float16,fp8,0,0.23426665862401327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,64,128,1,float16,float16,0,0.24174400170644125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,64,0,1,float16,float16,0,0.23685866594314575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,64,128,1,float16,fp8,0,0.24303466081619263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,64,0,1,float16,fp8,0,0.23478933175404867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,64,128,1,fp8,fp8,0,0.31753067175547284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,64,0,1,fp8,fp8,0,0.21176000436147055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,64,128,1,float16,float16,0,0.2500266631444295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,64,0,1,float16,float16,0,0.24247999986012778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,64,128,1,float16,fp8,0,0.24836266040802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,64,128,1,fp8,fp8,0,0.3197813431421916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,64,0,1,float16,fp8,0,0.24093866348266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,64,0,1,fp8,fp8,0,0.21311465899149576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,64,128,1,float16,float16,0,0.153519997994105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,64,0,1,float16,float16,0,0.15990933775901794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,64,128,1,float16,fp8,0,0.15125866731007895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,64,128,1,fp8,fp8,0,0.1869866649309794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,64,0,1,float16,fp8,0,0.1558026671409607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,64,0,1,fp8,fp8,0,0.12457600235939026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,64,128,1,float16,float16,0,0.1332319974899292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,64,0,1,fp8,fp8,0,0.11410666505495708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,64,0,1,float16,float16,0,0.1256053348382314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,64,128,1,float16,fp8,0,0.1318186620871226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,64,128,1,fp8,fp8,0,0.1744906703631083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,64,0,1,float16,fp8,0,0.1267306705315908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,64,0,1,float16,fp8,0,0.12760532895723978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,64,128,1,float16,float16,0,0.13455999890963236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,64,0,1,float16,float16,0,0.12641066312789917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,64,128,1,float16,fp8,0,0.1351146697998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,64,128,1,fp8,fp8,0,0.1778506636619568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,64,0,1,fp8,fp8,0,0.11598400274912517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,64,128,1,float16,float16,0,0.13691199819246927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,64,0,1,float16,float16,0,0.13084266583124796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,64,128,1,float16,fp8,0,0.1374666690826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,64,128,1,float16,fp8,0,0.09038933118184407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,64,128,1,fp8,fp8,0,0.17935999234517416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,64,0,1,float16,fp8,0,0.1299679974714915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,64,0,1,fp8,fp8,0,0.11839999755223592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,64,128,1,float16,float16,0,0.09083200494448344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,64,0,1,float16,float16,0,0.08258133133252461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,64,0,1,float16,float16,0,0.07134399811426799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,64,128,1,fp8,fp8,0,0.11301866173744202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,64,0,1,float16,fp8,0,0.08090666433175404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,64,0,1,fp8,fp8,0,0.07517866790294647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,64,0,1,fp8,fp8,0,0.06529066463311513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,64,128,1,float16,float16,0,0.07547200222810109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,64,128,1,float16,fp8,0,0.07633066674073537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,64,128,1,fp8,fp8,0,0.0986346701780955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,64,0,1,float16,fp8,0,0.07218666871388753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,64,128,1,float16,float16,0,0.07689066727956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,64,0,1,float16,float16,0,0.07233066856861115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,64,128,1,float16,fp8,0,0.07726400097211202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,64,128,1,fp8,fp8,0,0.10259200135866801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,64,0,1,float16,fp8,0,0.07248533268769582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,64,0,1,fp8,fp8,0,0.06572266419728597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,64,128,1,float16,float16,0,0.08075200021266937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,64,0,1,float16,float16,0,0.07458133498827617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,64,128,1,float16,fp8,0,0.08043733239173889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,64,128,1,fp8,fp8,0,0.10611200332641602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,64,0,1,float16,fp8,0,0.07436266541481018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,64,0,1,fp8,fp8,0,0.06846933563550313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,64,128,1,float16,float16,0,0.05351466437180837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,64,0,1,float16,float16,0,0.04829333225886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,64,128,1,float16,fp8,0,0.05227733155091604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,64,128,1,fp8,fp8,0,0.06909333169460297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,64,0,1,float16,fp8,0,0.047338664531707764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,64,0,1,fp8,fp8,0,0.043765331308046974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,64,128,1,float16,float16,0,0.04795733094215393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,64,0,1,float16,float16,0,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,64,128,1,float16,fp8,0,0.04818666477998098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,64,128,1,fp8,fp8,0,0.06113600234190623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,64,0,1,float16,fp8,0,0.043375998735427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,64,0,1,fp8,fp8,0,0.04043200115362803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,64,128,1,float16,float16,0,0.04752533137798309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,64,0,1,fp8,fp8,0,0.040336000422636666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,64,0,1,float16,float16,0,0.04383466641108195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,64,128,1,float16,fp8,0,0.048063998421033226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,64,128,1,fp8,fp8,0,0.06232533355553945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,64,0,1,float16,fp8,0,0.04489066700140635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,64,128,1,float16,float16,0,0.05041599770387014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,64,0,1,float16,float16,0,0.045066664616266884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,64,128,1,float16,fp8,0,0.05027199784914652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,64,128,1,fp8,fp8,0,0.06208533545335134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,64,0,1,float16,fp8,0,0.045594667394955955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,64,0,1,fp8,fp8,0,0.040949332217375435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,64,128,1,float16,float16,0,0.04046933352947235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,64,0,1,float16,float16,0,0.03479466587305069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,64,128,1,float16,fp8,0,0.03885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,64,128,1,fp8,fp8,0,0.042223999897638954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,64,0,1,float16,fp8,0,0.03477866699298223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,64,0,1,fp8,fp8,0,0.031301334500312805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,64,128,1,float16,float16,0,0.037615999579429626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,64,0,1,float16,float16,0,0.03257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,64,128,1,float16,fp8,0,0.035573333501815796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,64,128,1,fp8,fp8,0,0.040522667268911995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,64,0,1,float16,fp8,0,0.032272001107533775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,64,0,1,fp8,fp8,0,0.03014933317899704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,64,128,1,float16,float16,0,0.035717333356539406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,64,0,1,float16,float16,0,0.032485333581765495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,64,128,1,float16,fp8,0,0.03842133283615112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,64,128,1,fp8,fp8,0,0.041509332756201424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,64,0,1,float16,fp8,0,0.03252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,64,0,1,fp8,fp8,0,0.030576000610987347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,64,128,1,float16,float16,0,0.03656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,64,0,1,float16,float16,0,0.03311999887228012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,64,128,1,float16,fp8,0,0.03775999943415324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,64,128,1,fp8,fp8,0,0.041759997606277466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,64,0,1,float16,fp8,0,0.03386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,64,0,1,fp8,fp8,0,0.030847998956839245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,64,128,1,float16,float16,0,0.026213333010673523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,64,0,1,float16,float16,0,0.02446399877468745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,64,128,1,float16,fp8,0,0.026208000878492992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,64,128,1,fp8,fp8,0,0.03279466678698858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,64,0,1,float16,fp8,0,0.02380799998839696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,64,0,1,fp8,fp8,0,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,64,128,1,float16,float16,0,0.024874667326609295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,64,0,1,float16,float16,0,0.023418667415777843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,64,0,1,float16,float16,0,0.023269332945346832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,64,128,1,float16,fp8,0,0.025055999557177227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,64,128,1,fp8,fp8,0,0.03211733450492223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,64,0,1,float16,fp8,0,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,64,0,1,fp8,fp8,0,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,64,128,1,float16,float16,0,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,64,128,1,float16,float16,0,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,64,128,1,float16,fp8,0,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,64,128,1,fp8,fp8,0,0.03259733319282532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,64,0,1,float16,fp8,0,0.023589332898457844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,64,0,1,fp8,fp8,0,0.02221333235502243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,64,0,1,float16,float16,0,0.023077333966890972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,64,128,1,float16,fp8,0,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,64,128,1,float16,fp8,0,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,64,128,1,fp8,fp8,0,0.03206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,64,0,1,float16,fp8,0,0.023418667415777843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,64,0,1,fp8,fp8,0,0.0225600004196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,64,128,1,float16,float16,0,0.024682665864626568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,64,0,1,float16,float16,0,0.02266666789849599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,64,128,1,float16,fp8,0,0.023941333095232647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,64,128,1,fp8,fp8,0,0.030847998956839245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,64,0,1,float16,fp8,0,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,64,0,1,fp8,fp8,0,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,64,128,1,float16,float16,0,0.024485332270463307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,64,0,1,float16,float16,0,0.022074667116006214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,64,128,1,fp8,fp8,0,0.031199999153614044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,64,0,1,float16,fp8,0,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,64,0,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,64,128,1,float16,float16,0,0.024101334313551586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,64,0,1,fp8,fp8,0,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,64,128,1,float16,float16,0,0.024501333634058636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,64,0,1,float16,float16,0,0.02203733225663503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,64,128,1,float16,fp8,0,0.02447466552257538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,64,128,1,fp8,fp8,0,0.030586667358875275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,64,0,1,float16,fp8,0,0.022976001103719074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,64,0,1,float16,float16,0,0.022485333184401195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,64,128,1,float16,fp8,0,0.024341332415739696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,64,128,1,fp8,fp8,0,0.030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,64,0,1,float16,fp8,0,0.022367998957633972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,64,0,1,fp8,fp8,0,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,64,128,1,float16,float16,0,0.3824160099029541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,64,0,1,float16,float16,0,0.37277865409851074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,64,128,1,float16,fp8,0,0.3803946574529012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,64,0,1,float16,fp8,0,0.37109867731730145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,64,128,1,fp8,fp8,0,0.5072746674219767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,64,0,1,fp8,fp8,0,0.34055999914805096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,64,128,1,float16,float16,0,0.3858826557795207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,64,0,1,float16,float16,0,0.3758666515350342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,64,128,1,float16,fp8,0,0.3818986813227336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,64,128,1,fp8,fp8,0,0.518336017926534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,64,0,1,float16,fp8,0,0.37304532527923584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,64,0,1,fp8,fp8,0,0.34218132495880127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,64,128,1,float16,float16,0,0.3984053134918213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,64,0,1,float16,float16,0,0.38847466309865314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,64,128,1,float16,fp8,0,0.39587199687957764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,64,0,1,float16,fp8,0,0.38709867000579834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,64,128,1,fp8,fp8,0,0.538319985071818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,64,128,1,float16,float16,0,0.2526560028394063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,64,0,1,fp8,fp8,0,0.3707733154296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,64,0,1,float16,float16,0,0.24913599093755087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,64,128,1,float16,fp8,0,0.24492265780766806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,64,128,1,fp8,fp8,0,0.2869493365287781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,64,0,1,float16,fp8,0,0.24100800355275473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,64,0,1,fp8,fp8,0,0.20332799355189005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,64,128,1,float16,float16,0,0.202239990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,64,0,1,float16,float16,0,0.1981226603190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,64,128,1,float16,fp8,0,0.20287466049194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,64,128,1,fp8,fp8,0,0.26576000452041626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,64,0,1,float16,fp8,0,0.1971679925918579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,64,0,1,fp8,fp8,0,0.18237866957982382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,64,128,1,float16,float16,0,0.20433600743611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,64,0,1,float16,float16,0,0.2002240022023519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,64,128,1,float16,fp8,0,0.20385066668192545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,64,128,1,fp8,fp8,0,0.2621226708094279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,64,0,1,fp8,fp8,0,0.18101332585016885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,64,0,1,float16,fp8,0,0.1994826594988505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,64,128,1,float16,float16,0,0.21370132764180502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,64,128,1,float16,float16,0,0.14043200016021729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,64,0,1,float16,float16,0,0.2095253268877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,64,128,1,float16,fp8,0,0.21127466360727945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,64,128,1,fp8,fp8,0,0.2754773298899333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,64,0,1,float16,fp8,0,0.20865066846211752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,64,0,1,fp8,fp8,0,0.19350399573644003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,64,0,1,float16,float16,0,0.13933866222699484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,64,128,1,float16,fp8,0,0.13684266805648804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,64,128,1,fp8,fp8,0,0.15436266859372458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,64,0,1,float16,fp8,0,0.13500266273816428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,64,0,1,fp8,fp8,0,0.11425600449244182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,64,128,1,float16,float16,0,0.10797333717346191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,64,0,1,float16,float16,0,0.10521066188812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,64,128,1,float16,fp8,0,0.10763733585675557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,64,128,1,fp8,fp8,0,0.1433013379573822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,64,0,1,float16,fp8,0,0.1051093339920044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,64,0,1,fp8,fp8,0,0.1018453339735667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,64,128,1,float16,float16,0,0.11126933495203654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,64,0,1,float16,float16,0,0.10947199662526448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,64,128,1,float16,fp8,0,0.11000000437100728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,64,128,1,fp8,fp8,0,0.14258133371671042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,64,0,1,float16,fp8,0,0.10822400450706482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,64,128,1,fp8,fp8,0,0.146314670642217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,64,0,1,fp8,fp8,0,0.10205333431561787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,64,128,1,float16,float16,0,0.11839466293652852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,64,0,1,float16,float16,0,0.11692800124486287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,64,128,1,float16,fp8,0,0.11706667145093282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,64,0,1,float16,fp8,0,0.11572800079981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,64,0,1,fp8,fp8,0,0.10684800148010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,64,128,1,float16,float16,0,0.08379200100898743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,64,0,1,float16,float16,0,0.08260799944400787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,64,128,1,float16,fp8,0,0.08190399905045827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,64,128,1,fp8,fp8,0,0.08806399504343669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,64,0,1,float16,fp8,0,0.08054933448632558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,64,0,1,fp8,fp8,0,0.06681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,64,128,1,float16,float16,0,0.06283733248710632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,64,0,1,float16,float16,0,0.062362665931383766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,64,128,1,float16,fp8,0,0.06325866778691609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,64,128,1,fp8,fp8,0,0.07786133388678233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,64,0,1,float16,fp8,0,0.06359999875227611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,64,0,1,float16,fp8,0,0.06366399923960368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,64,0,1,fp8,fp8,0,0.057909334699312844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,64,128,1,float16,float16,0,0.06852266689141591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,64,128,1,float16,float16,0,0.06482666730880737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,64,0,1,float16,float16,0,0.06454933186372121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,64,128,1,float16,fp8,0,0.06458133459091187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,64,128,1,fp8,fp8,0,0.07919999957084656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,64,0,1,fp8,fp8,0,0.061119998494784035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,64,0,1,fp8,fp8,0,0.06150400141874949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,64,0,1,float16,float16,0,0.06742933392524719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,64,128,1,float16,fp8,0,0.06687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,64,128,1,fp8,fp8,0,0.0831413318713506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,64,0,1,float16,fp8,0,0.06764266888300578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,64,128,1,float16,float16,0,0.042447999119758606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,64,0,1,float16,float16,0,0.04074666649103165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,64,128,1,float16,fp8,0,0.041237334410349526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,64,128,1,fp8,fp8,0,0.050800000627835594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,64,0,1,float16,fp8,0,0.040991999208927155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,64,0,1,fp8,fp8,0,0.039520000418027244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,64,128,1,float16,float16,0,0.03807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,64,0,1,float16,float16,0,0.037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,64,128,1,float16,float16,0,0.03827200084924698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,64,128,1,float16,fp8,0,0.03836799909671148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,64,0,1,float16,float16,0,0.0377866675456365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,64,128,1,fp8,fp8,0,0.04537599782148997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,64,0,1,float16,fp8,0,0.0371573343873024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,64,0,1,fp8,fp8,0,0.035573333501815796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,64,128,1,float16,fp8,0,0.03867200016975403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,64,128,1,fp8,fp8,0,0.04666133224964142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,64,0,1,float16,fp8,0,0.03799466788768768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,64,0,1,fp8,fp8,0,0.036464000741640724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,64,128,1,float16,float16,0,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,64,0,1,float16,float16,0,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,64,128,1,float16,fp8,0,0.03982933362325033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,64,128,1,fp8,fp8,0,0.04711466530958811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,64,0,1,float16,fp8,0,0.03852266569932302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,64,0,1,fp8,fp8,0,0.0371573343873024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,64,128,1,float16,float16,0,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,64,0,1,float16,float16,0,0.031040000418821972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,64,128,1,float16,fp8,0,0.03155199935038885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,64,128,1,fp8,fp8,0,0.03608533243338267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,64,0,1,float16,fp8,0,0.030917334059874218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,64,0,1,fp8,fp8,0,0.029066666960716248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,64,128,1,float16,float16,0,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,64,0,1,float16,float16,0,0.028560000161329906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,64,128,1,float16,float16,0,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,64,128,1,float16,fp8,0,0.02900800108909607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,64,128,1,fp8,fp8,0,0.034485332667827606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,64,0,1,float16,fp8,0,0.02846933404604594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,64,0,1,fp8,fp8,0,0.02769600103298823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,64,0,1,float16,float16,0,0.029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,64,128,1,float16,fp8,0,0.029722665747006733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,64,128,1,fp8,fp8,0,0.035386666655540466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,64,0,1,float16,fp8,0,0.029071999092896778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,64,128,1,fp8,fp8,0,0.03596800069014231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,64,0,1,fp8,fp8,0,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,64,128,1,float16,float16,0,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,64,0,1,float16,float16,0,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,64,128,1,float16,fp8,0,0.030789333085219067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,64,0,1,float16,fp8,0,0.03012266755104065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,64,0,1,fp8,fp8,0,0.028560000161329906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,64,128,1,float16,float16,0,0.020367999871571858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,64,0,1,float16,float16,0,0.020047999918460846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,64,128,1,float16,fp8,0,0.020234666764736176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,64,128,1,fp8,fp8,0,0.02439466615517934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,64,0,1,float16,fp8,0,0.019695999721686046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,64,0,1,fp8,fp8,0,0.02085866779088974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,64,128,1,float16,float16,0,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,64,0,1,float16,float16,0,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,64,128,1,float16,fp8,0,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,64,128,1,fp8,fp8,0,0.022991999983787537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,64,0,1,float16,fp8,0,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,64,0,1,fp8,fp8,0,0.020047999918460846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,64,128,1,float16,float16,0,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,64,0,1,float16,float16,0,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,64,128,1,float16,fp8,0,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,64,128,1,fp8,fp8,0,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,64,0,1,float16,fp8,0,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,64,0,1,fp8,fp8,0,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,64,128,1,float16,float16,0,0.019199999670187633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,64,0,1,float16,float16,0,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,64,128,1,float16,fp8,0,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,64,128,1,fp8,fp8,0,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,64,0,1,float16,fp8,0,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,64,0,1,fp8,fp8,0,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,64,0,1,fp8,fp8,0,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,64,128,1,float16,float16,0,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,64,0,1,float16,float16,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,64,128,1,float16,fp8,0,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,64,128,1,fp8,fp8,0,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,64,0,1,float16,fp8,0,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,64,128,1,float16,float16,0,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,64,0,1,float16,float16,0,0.015978666643301647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,64,128,1,float16,fp8,0,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,64,128,1,fp8,fp8,0,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,64,0,1,float16,fp8,0,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,64,0,1,float16,fp8,0,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,64,0,1,fp8,fp8,0,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,64,128,1,float16,float16,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,64,0,1,float16,float16,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,64,128,1,float16,fp8,0,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,64,128,1,fp8,fp8,0,0.022064000368118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,64,0,1,fp8,fp8,0,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,64,0,1,fp8,fp8,0,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,64,128,1,float16,float16,0,0.016336000214020412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,64,0,1,float16,float16,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,64,128,1,float16,fp8,0,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,64,128,1,fp8,fp8,0,0.02254933367172877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,64,128,1,fp8,fp8,0,0.021967999637126923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,64,0,1,float16,fp8,0,0.016704000532627106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,64,128,1,float16,float16,0,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,64,0,1,float16,float16,0,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,64,128,1,float16,fp8,0,0.015919999529918034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,64,0,1,float16,fp8,0,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,64,128,1,fp8,fp8,0,0.021722666919231415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,64,0,1,float16,fp8,0,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,64,0,1,fp8,fp8,0,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,64,128,1,float16,float16,0,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,64,0,1,float16,float16,0,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,64,128,1,float16,fp8,0,0.015930666277805965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,64,128,1,fp8,fp8,0,0.021722666919231415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,64,0,1,fp8,fp8,0,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,64,128,1,float16,float16,0,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,64,0,1,float16,float16,0,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,64,128,1,float16,fp8,0,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,64,0,1,float16,fp8,0,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,64,128,1,fp8,fp8,0,0.021903999149799347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,64,0,1,fp8,fp8,0,0.018165333817402523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,64,128,1,float16,float16,0,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,64,0,1,float16,float16,0,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,64,128,1,float16,fp8,0,0.01605333387851715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,64,128,1,float16,float16,0,0.14934399724006653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,64,0,1,float16,fp8,0,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,64,0,1,fp8,fp8,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,64,0,1,float16,float16,0,0.14866133530934653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,64,128,1,float16,fp8,0,0.14781866470972696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,64,128,1,fp8,fp8,0,0.1803200046221415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,64,0,1,float16,fp8,0,0.14801067113876343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,64,0,1,fp8,fp8,0,0.17863466342290243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,64,128,1,float16,float16,0,0.15185067057609558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,64,0,1,float16,float16,0,0.1515679955482483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,64,128,1,float16,fp8,0,0.15017599860827127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,64,128,1,fp8,fp8,0,0.18405866622924805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,64,0,1,float16,fp8,0,0.1502400040626526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,64,0,1,fp8,fp8,0,0.18316799402236938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,64,128,1,float16,float16,0,0.15819733341534933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,64,0,1,float16,float16,0,0.1588053305943807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,64,128,1,float16,fp8,0,0.15745066603024802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,64,128,1,fp8,fp8,0,0.19405333201090494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,64,0,1,float16,fp8,0,0.1564586659272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,64,128,1,float16,float16,0,0.11343999703725179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,64,0,1,float16,float16,0,0.1141973336537679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,64,0,1,fp8,fp8,0,0.19307732582092285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,64,128,1,float16,fp8,0,0.11143466830253601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,64,128,1,fp8,fp8,0,0.11343466242154439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,64,0,1,float16,fp8,0,0.11018666625022888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,64,0,1,fp8,fp8,0,0.11307733257611592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,64,128,1,float16,float16,0,0.0823413332303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,64,0,1,float16,float16,0,0.08191466828187306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,64,128,1,float16,fp8,0,0.08182933429876964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,64,128,1,fp8,fp8,0,0.10020266969998677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,64,0,1,float16,fp8,0,0.08205866813659668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,64,0,1,fp8,fp8,0,0.10012800494829814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,64,128,1,float16,float16,0,0.08490133285522461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,64,0,1,float16,float16,0,0.0843946635723114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,64,128,1,float16,fp8,0,0.08488000432650249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,64,128,1,fp8,fp8,0,0.1013759970664978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,64,0,1,float16,fp8,0,0.08458133538564046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,64,128,1,fp8,fp8,0,0.10549333691596985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,64,0,1,fp8,fp8,0,0.1013759970664978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,64,128,1,float16,float16,0,0.08910399675369263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,64,0,1,float16,float16,0,0.08988799651463826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,64,128,1,float16,fp8,0,0.08885332942008972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,64,0,1,float16,fp8,0,0.08964266379674275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,64,0,1,float16,fp8,0,0.062165334820747375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,64,0,1,fp8,fp8,0,0.10549333691596985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,64,128,1,float16,float16,0,0.06387733419736226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,64,0,1,float16,float16,0,0.0639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,64,128,1,float16,fp8,0,0.06102400024731954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,64,128,1,fp8,fp8,0,0.06623999774456024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,64,0,1,fp8,fp8,0,0.06655466556549072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,64,128,1,float16,float16,0,0.04728533327579498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,64,0,1,float16,float16,0,0.04693866769472758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,64,128,1,float16,fp8,0,0.046485334634780884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,64,128,1,fp8,fp8,0,0.05651199817657471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,64,0,1,float16,fp8,0,0.04621333380540212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,64,0,1,fp8,fp8,0,0.05717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,64,128,1,float16,float16,0,0.047968000173568726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,64,0,1,float16,float16,0,0.047930667797724404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,64,128,1,float16,fp8,0,0.0476693312327067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,64,128,1,fp8,fp8,0,0.05938666562239329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,64,0,1,float16,fp8,0,0.048063998421033226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,64,0,1,fp8,fp8,0,0.05829333265622457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,64,128,1,float16,float16,0,0.050111999114354454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,64,0,1,float16,float16,0,0.04993066688378652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,64,128,1,float16,fp8,0,0.05019199848175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,64,128,1,fp8,fp8,0,0.06145599981149038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,64,0,1,float16,fp8,0,0.049733335773150124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,64,0,1,fp8,fp8,0,0.061381335059801735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,64,128,1,float16,float16,0,0.03164800008138021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,64,0,1,float16,float16,0,0.03245333333810171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,64,128,1,float16,fp8,0,0.03136000037193298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,64,128,1,fp8,fp8,0,0.038592000802357994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,64,0,1,float16,fp8,0,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,64,0,1,fp8,fp8,0,0.03929600119590759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,64,128,1,float16,float16,0,0.02804800122976303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,64,0,1,float16,float16,0,0.02794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,64,128,1,float16,fp8,0,0.028336000939210255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,64,128,1,fp8,fp8,0,0.03453333427508672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,64,0,1,float16,fp8,0,0.02807466685771942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,64,0,1,fp8,fp8,0,0.035562666753927864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,64,128,1,float16,float16,0,0.02868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,64,0,1,fp8,fp8,0,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,64,128,1,float16,float16,0,0.029946667452653248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,64,0,1,float16,float16,0,0.028490667541821797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,64,128,1,float16,fp8,0,0.028357334434986115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,64,128,1,fp8,fp8,0,0.03579733272393545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,64,0,1,float16,fp8,0,0.027877333263556164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,64,0,1,float16,float16,0,0.029765332738558452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,64,128,1,float16,fp8,0,0.02922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,64,128,1,fp8,fp8,0,0.036559998989105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,64,0,1,float16,fp8,0,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,64,0,1,fp8,fp8,0,0.03640533238649368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,64,128,1,float16,float16,0,0.024218666056791942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,64,0,1,fp8,fp8,0,0.02844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,64,0,1,float16,float16,0,0.024117333193620045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,64,128,1,float16,fp8,0,0.023818666736284893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,64,128,1,fp8,fp8,0,0.028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,64,0,1,float16,fp8,0,0.02366400013367335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,64,128,1,float16,float16,0,0.022245332598686218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,64,0,1,float16,float16,0,0.022069332500298817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,64,128,1,float16,fp8,0,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,64,128,1,fp8,fp8,0,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,64,0,1,float16,fp8,0,0.022096000611782074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,64,0,1,fp8,fp8,0,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,64,128,1,float16,float16,0,0.022133332987626392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,64,0,1,float16,float16,0,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,64,128,1,float16,fp8,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,64,128,1,fp8,fp8,0,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,64,0,1,float16,fp8,0,0.02276800076166789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,64,0,1,fp8,fp8,0,0.028149334092934925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,64,128,1,float16,float16,0,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,64,0,1,float16,float16,0,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,64,128,1,float16,fp8,0,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,64,128,1,fp8,fp8,0,0.028384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,64,0,1,float16,fp8,0,0.022800001005331676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,64,0,1,fp8,fp8,0,0.028565332293510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,64,128,1,float16,float16,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,64,0,1,float16,float16,0,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,64,128,1,float16,fp8,0,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,64,128,1,fp8,fp8,0,0.020554666717847187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,64,0,1,float16,fp8,0,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,64,0,1,fp8,fp8,0,0.020175999651352566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,64,128,1,float16,float16,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,64,0,1,float16,float16,0,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,64,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,64,128,1,float16,fp8,0,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,64,0,1,float16,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,64,128,1,fp8,fp8,0,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,64,0,1,fp8,fp8,0,0.019706666469573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,64,128,1,float16,float16,0,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,64,128,1,float16,fp8,0,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,64,128,1,fp8,fp8,0,0.020165332903464634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,64,0,1,float16,fp8,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,64,0,1,fp8,fp8,0,0.019679999599854153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,64,128,1,float16,float16,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,64,0,1,float16,float16,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,64,128,1,float16,fp8,0,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,64,128,1,fp8,fp8,0,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,64,0,1,float16,fp8,0,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,64,0,1,fp8,fp8,0,0.020341333001852036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,64,128,1,float16,float16,0,0.014426667243242264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,64,0,1,float16,float16,0,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,64,128,1,float16,fp8,0,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,64,128,1,fp8,fp8,0,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,64,0,1,float16,fp8,0,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,64,0,1,fp8,fp8,0,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,64,128,1,float16,float16,0,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,64,0,1,float16,float16,0,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,64,128,1,float16,fp8,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,64,128,1,fp8,fp8,0,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,64,0,1,float16,fp8,0,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,64,0,1,fp8,fp8,0,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,64,128,1,float16,float16,0,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,64,0,1,float16,float16,0,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,64,128,1,float16,fp8,0,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,64,128,1,fp8,fp8,0,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,64,0,1,float16,fp8,0,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,64,0,1,fp8,fp8,0,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,64,128,1,float16,float16,0,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,64,0,1,float16,float16,0,0.014261333892742792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,64,0,1,float16,float16,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,64,128,1,float16,fp8,0,0.014389333625634512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,64,128,1,fp8,fp8,0,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,64,0,1,float16,fp8,0,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,64,0,1,fp8,fp8,0,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,64,128,1,float16,float16,0,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,64,128,1,float16,fp8,0,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,64,128,1,fp8,fp8,0,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,64,0,1,float16,fp8,0,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,64,0,1,fp8,fp8,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,64,128,1,float16,float16,0,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,64,0,1,float16,float16,0,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,64,128,1,float16,fp8,0,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,64,128,1,fp8,fp8,0,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,64,0,1,float16,fp8,0,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,64,0,1,fp8,fp8,0,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,64,128,1,float16,float16,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,64,0,1,float16,float16,0,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,64,128,1,float16,fp8,0,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,64,128,1,fp8,fp8,0,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,64,0,1,float16,fp8,0,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,64,0,1,fp8,fp8,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,64,128,1,float16,float16,0,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,64,0,1,float16,float16,0,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,64,128,1,float16,float16,0,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,64,128,1,float16,fp8,0,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,64,128,1,fp8,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,64,0,1,float16,fp8,0,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,64,0,1,fp8,fp8,0,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,64,0,1,float16,float16,0,0.012650666137536367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,64,128,1,float16,fp8,0,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,64,128,1,fp8,fp8,0,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,64,0,1,float16,fp8,0,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,64,0,1,fp8,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,64,128,1,float16,float16,0,0.012608000387748083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,64,0,1,float16,float16,0,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,64,128,1,float16,fp8,0,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,64,128,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,64,0,1,float16,fp8,0,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,64,0,1,fp8,fp8,0,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,64,0,1,float16,fp8,0,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,64,128,1,float16,float16,0,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,64,0,1,float16,float16,0,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,64,128,1,float16,fp8,0,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,64,128,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,64,128,1,fp8,fp8,0,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,64,0,1,fp8,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,64,128,1,float16,float16,0,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,64,0,1,float16,float16,0,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,64,128,1,float16,fp8,0,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,64,0,1,float16,fp8,0,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,64,0,1,fp8,fp8,0,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,64,128,1,float16,float16,0,0.0792799989382426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,64,0,1,float16,float16,0,0.07830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,64,128,1,fp8,fp8,0,0.13201066851615906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,64,128,1,float16,fp8,0,0.07881066699822743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,64,0,1,fp8,fp8,0,0.1312266687552134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,64,0,1,float16,fp8,0,0.07807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,64,128,1,float16,float16,0,0.0821919987599055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,64,0,1,float16,float16,0,0.08156799773375194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,64,128,1,float16,fp8,0,0.08044266700744629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,64,128,1,fp8,fp8,0,0.13246933619181314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,64,0,1,float16,fp8,0,0.07998933394749959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,64,0,1,fp8,fp8,0,0.13195199767748514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,64,128,1,float16,float16,0,0.08409066994984944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,64,0,1,float16,float16,0,0.08472533027331035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,64,128,1,float16,fp8,0,0.08435199658075969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,64,128,1,fp8,fp8,0,0.13777066270510355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,64,0,1,float16,fp8,0,0.08438400427500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,64,128,1,float16,float16,0,0.06136533121267954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,64,0,1,fp8,fp8,0,0.13689066966374716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,64,0,1,float16,float16,0,0.0610453337430954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,64,128,1,float16,float16,0,0.044351999958356224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,64,128,1,float16,fp8,0,0.058464000622431435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,64,128,1,fp8,fp8,0,0.08347733815511067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,64,0,1,float16,fp8,0,0.05840533475081126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,64,0,1,fp8,fp8,0,0.082805335521698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,64,0,1,float16,float16,0,0.04438933233420054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,64,128,1,float16,float16,0,0.04555733501911163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,64,128,1,float16,fp8,0,0.044810667634010315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,64,128,1,fp8,fp8,0,0.07206933200359344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,64,0,1,float16,fp8,0,0.044735997915267944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,64,0,1,fp8,fp8,0,0.07245866457621257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,64,0,1,float16,float16,0,0.04471466441949209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,64,128,1,float16,fp8,0,0.045141334335009255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,64,128,1,fp8,fp8,0,0.0759626676638921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,64,0,1,float16,fp8,0,0.044981335600217186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,64,0,1,fp8,fp8,0,0.07588266829649608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,64,128,1,float16,float16,0,0.04743466774622599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,64,0,1,float16,float16,0,0.04795200129350027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,64,128,1,float16,fp8,0,0.047168001532554626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,64,128,1,fp8,fp8,0,0.07814399898052216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,64,0,1,float16,fp8,0,0.04741866886615753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,64,128,1,float16,float16,0,0.030005333324273426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,64,0,1,fp8,fp8,0,0.07781333227952321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,64,0,1,float16,float16,0,0.02977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,64,128,1,float16,fp8,0,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,64,128,1,fp8,fp8,0,0.04719999929269155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,64,0,1,float16,fp8,0,0.029333333174387615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,64,0,1,fp8,fp8,0,0.04698666433493296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,64,128,1,float16,float16,0,0.02678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,64,0,1,float16,float16,0,0.0262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,64,128,1,float16,fp8,0,0.027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,64,128,1,fp8,fp8,0,0.043365334471066795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,64,0,1,float16,fp8,0,0.026821332673231762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,64,0,1,fp8,fp8,0,0.0432640016078949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,64,0,1,fp8,fp8,0,0.04393066465854645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,64,128,1,float16,float16,0,0.026816000541051228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,64,0,1,float16,float16,0,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,64,128,1,float16,fp8,0,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,64,128,1,fp8,fp8,0,0.04363200068473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,64,0,1,float16,fp8,0,0.027002667387326557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,64,128,1,float16,float16,0,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,64,0,1,float16,float16,0,0.027482666075229645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,64,128,1,float16,fp8,0,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,64,128,1,fp8,fp8,0,0.0444160004456838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,64,0,1,float16,fp8,0,0.02805333336194356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,64,0,1,fp8,fp8,0,0.044533332188924156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,64,128,1,float16,float16,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,64,0,1,float16,float16,0,0.02176533391078313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,64,128,1,float16,fp8,0,0.021536000072956085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,64,128,1,fp8,fp8,0,0.030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,64,0,1,float16,fp8,0,0.02179733415444692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,64,0,1,fp8,fp8,0,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,64,128,1,float16,float16,0,0.020501332978407543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,64,0,1,float16,float16,0,0.02033599962790807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,64,128,1,float16,fp8,0,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,64,128,1,fp8,fp8,0,0.02975466599067052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,64,0,1,float16,fp8,0,0.02054399996995926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,64,0,1,fp8,fp8,0,0.029738667110602062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,64,128,1,float16,float16,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,64,0,1,float16,float16,0,0.020869334538777668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,64,128,1,float16,float16,0,0.021477334201335907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,64,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,64,128,1,float16,fp8,0,0.020773333807786305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,64,128,1,fp8,fp8,0,0.029994666576385498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,64,0,1,float16,fp8,0,0.021477334201335907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,64,0,1,float16,fp8,0,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,64,0,1,fp8,fp8,0,0.03013866643110911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,64,128,1,float16,fp8,0,0.02197866638501485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,64,128,1,fp8,fp8,0,0.031173333525657654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,64,0,1,fp8,fp8,0,0.030645333230495453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,64,128,1,float16,float16,0,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,64,0,1,fp8,fp8,0,0.023738667368888855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,64,0,1,float16,float16,0,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,64,128,1,float16,fp8,0,0.015605332950750986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,64,128,1,fp8,fp8,0,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,64,0,1,float16,fp8,0,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,64,128,1,float16,float16,0,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,64,0,1,float16,float16,0,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,64,128,1,float16,fp8,0,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,64,128,1,fp8,fp8,0,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,64,0,1,float16,fp8,0,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,64,0,1,fp8,fp8,0,0.02327466756105423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,64,0,1,float16,fp8,0,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,64,128,1,float16,float16,0,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,64,0,1,float16,float16,0,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,64,128,1,float16,fp8,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,64,128,1,float16,fp8,0,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,64,128,1,fp8,fp8,0,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,64,0,1,fp8,fp8,0,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,64,128,1,float16,float16,0,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,64,128,1,float16,float16,0,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,64,0,1,float16,float16,0,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,64,128,1,fp8,fp8,0,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,64,0,1,float16,fp8,0,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,64,0,1,float16,fp8,0,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,64,0,1,fp8,fp8,0,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,64,0,1,float16,float16,0,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,64,128,1,float16,fp8,0,0.01357866699496905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,64,128,1,fp8,fp8,0,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,64,0,1,fp8,fp8,0,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,64,128,1,float16,float16,0,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,64,0,1,float16,float16,0,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,64,128,1,float16,fp8,0,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,64,128,1,fp8,fp8,0,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,64,0,1,float16,fp8,0,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,64,0,1,fp8,fp8,0,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,64,128,1,float16,float16,0,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,64,0,1,float16,float16,0,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,64,128,1,float16,fp8,0,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,64,128,1,fp8,fp8,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,64,0,1,float16,fp8,0,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,64,0,1,fp8,fp8,0,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,64,128,1,float16,float16,0,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,64,0,1,float16,float16,0,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,64,128,1,float16,fp8,0,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,64,128,1,fp8,fp8,0,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,64,0,1,float16,fp8,0,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,64,0,1,fp8,fp8,0,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,64,0,1,fp8,fp8,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,64,128,1,float16,float16,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,64,128,1,float16,fp8,0,0.012245333443085352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,64,0,1,float16,float16,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,64,128,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,64,0,1,float16,fp8,0,0.012298667182525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,64,128,1,float16,float16,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,64,128,1,float16,float16,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,64,0,1,float16,float16,0,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,64,128,1,float16,fp8,0,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,64,128,1,fp8,fp8,0,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,64,0,1,float16,fp8,0,0.012527999778588613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,64,0,1,fp8,fp8,0,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,64,0,1,fp8,fp8,0,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,64,0,1,float16,float16,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,64,128,1,float16,fp8,0,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,64,128,1,fp8,fp8,0,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,64,128,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,64,0,1,float16,fp8,0,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,64,128,1,float16,float16,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,64,0,1,float16,float16,0,0.012223999947309494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,64,128,1,float16,fp8,0,0.01258133351802826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,64,0,1,float16,fp8,0,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,64,0,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,64,0,1,float16,fp8,0,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,64,128,1,float16,float16,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,64,0,1,float16,float16,0,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,64,128,1,float16,fp8,0,0.011946666985750198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,64,128,1,float16,fp8,0,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,64,128,1,fp8,fp8,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,64,0,1,fp8,fp8,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,64,128,1,float16,float16,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,64,0,1,float16,float16,0,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,64,128,1,fp8,fp8,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,64,0,1,float16,fp8,0,0.012117333710193634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,64,128,1,fp8,fp8,0,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,64,0,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,64,128,1,float16,float16,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,64,0,1,float16,float16,0,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,64,128,1,float16,fp8,0,0.012080000092585882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,64,0,1,float16,fp8,0,0.01221866657336553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,64,0,1,fp8,fp8,0,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,64,128,1,float16,float16,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,64,0,1,float16,float16,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,64,128,1,float16,fp8,0,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,64,128,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,64,0,1,float16,fp8,0,0.012266666938861212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,64,0,1,fp8,fp8,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,64,128,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,64,0,1,float16,float16,0,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,64,128,1,float16,fp8,0,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,64,128,1,fp8,fp8,0,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,64,0,1,float16,fp8,0,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,64,0,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,64,128,1,float16,float16,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,64,0,1,float16,float16,0,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,64,128,1,float16,fp8,0,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,64,128,1,fp8,fp8,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,64,0,1,float16,fp8,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,64,0,1,fp8,fp8,0,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,64,128,1,float16,float16,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,64,0,1,float16,float16,0,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,64,128,1,float16,fp8,0,0.01227733368674914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,64,128,1,fp8,fp8,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,64,0,1,float16,fp8,0,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,64,0,1,fp8,fp8,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,64,0,1,fp8,fp8,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,64,128,1,float16,float16,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,64,0,1,float16,float16,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,64,128,1,float16,fp8,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,64,128,1,fp8,fp8,0,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,64,0,1,float16,fp8,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,64,128,1,float16,float16,0,0.04613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,64,0,1,float16,float16,0,0.04580266773700714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,64,128,1,float16,fp8,0,0.04557333389918009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,64,128,1,fp8,fp8,0,0.10750400026639302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,64,0,1,float16,fp8,0,0.046015997727712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,64,0,1,fp8,fp8,0,0.10772266983985901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,64,128,1,float16,float16,0,0.047295997540156044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,64,0,1,float16,float16,0,0.04740266501903534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,64,128,1,float16,fp8,0,0.047184000412623085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,64,128,1,fp8,fp8,0,0.1097866694132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,64,128,1,float16,float16,0,0.04966400067011515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,64,0,1,float16,fp8,0,0.04655466477076212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,64,0,1,fp8,fp8,0,0.11042666435241699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,64,0,1,float16,float16,0,0.048986668388048805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,64,128,1,float16,fp8,0,0.04876266419887543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,64,128,1,fp8,fp8,0,0.11306132872899373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,64,0,1,float16,fp8,0,0.0489279975493749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,64,128,1,float16,float16,0,0.03035199890534083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,64,0,1,float16,float16,0,0.030250666042168934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,64,0,1,fp8,fp8,0,0.11205333471298218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,64,128,1,float16,fp8,0,0.0296426663796107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,64,128,1,fp8,fp8,0,0.06646933158238728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,64,0,1,float16,fp8,0,0.029333333174387615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,64,0,1,fp8,fp8,0,0.06653333206971486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,64,128,1,float16,float16,0,0.027973333994547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,64,0,1,float16,float16,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,64,128,1,float16,fp8,0,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,64,128,1,fp8,fp8,0,0.06180266539255778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,64,0,1,float16,fp8,0,0.02788266787926356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,64,0,1,fp8,fp8,0,0.06211733321348826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,64,128,1,float16,float16,0,0.028069332242012024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,64,128,1,float16,float16,0,0.028725333511829376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,64,0,1,float16,float16,0,0.028031999866167705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,64,128,1,float16,fp8,0,0.028245332340399425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,64,128,1,fp8,fp8,0,0.06365333497524261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,64,0,1,float16,fp8,0,0.027957332630952198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,64,0,1,fp8,fp8,0,0.06266133487224579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,64,0,1,float16,float16,0,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,64,128,1,float16,fp8,0,0.029066666960716248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,64,128,1,fp8,fp8,0,0.06353066861629486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,64,0,1,float16,fp8,0,0.028773332635561626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,64,0,1,fp8,fp8,0,0.06377600133419037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,64,128,1,float16,float16,0,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,64,0,1,float16,float16,0,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,64,128,1,float16,fp8,0,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,64,128,1,fp8,fp8,0,0.04074133435885111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,64,128,1,fp8,fp8,0,0.03965333352486292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,64,0,1,float16,fp8,0,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,64,0,1,fp8,fp8,0,0.04092800120512644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,64,128,1,float16,float16,0,0.01993600030740102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,64,0,1,float16,float16,0,0.02000533292690913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,64,128,1,float16,fp8,0,0.01988799994190534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,64,0,1,float16,fp8,0,0.020165332903464634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,64,0,1,fp8,fp8,0,0.03990400085846583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,64,128,1,float16,float16,0,0.019968000551064808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,64,0,1,float16,float16,0,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,64,0,1,fp8,fp8,0,0.040362666050593056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,64,128,1,float16,fp8,0,0.020608000457286835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,64,128,1,fp8,fp8,0,0.04031999905904134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,64,0,1,float16,fp8,0,0.02014933278163274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,64,128,1,float16,float16,0,0.020608000457286835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,64,0,1,float16,float16,0,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,64,128,1,float16,fp8,0,0.02082666630546252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,64,128,1,fp8,fp8,0,0.04049599915742874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,64,0,1,float16,fp8,0,0.020853333175182343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,64,0,1,fp8,fp8,0,0.04071466624736786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,64,128,1,float16,float16,0,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,64,0,1,fp8,fp8,0,0.026528000831604004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,64,0,1,float16,float16,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,64,128,1,float16,fp8,0,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,64,128,1,fp8,fp8,0,0.026208000878492992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,64,0,1,float16,fp8,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,64,128,1,float16,float16,0,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,64,0,1,float16,float16,0,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,64,128,1,float16,fp8,0,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,64,128,1,fp8,fp8,0,0.025839999318122864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,64,0,1,float16,fp8,0,0.014650666465361914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,64,0,1,fp8,fp8,0,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,64,128,1,float16,float16,0,0.014117332796255747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,64,0,1,float16,float16,0,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,64,128,1,float16,fp8,0,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,64,128,1,fp8,fp8,0,0.025818665822347004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,64,0,1,float16,fp8,0,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,64,0,1,fp8,fp8,0,0.026159999271233875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,64,128,1,float16,float16,0,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,64,0,1,float16,float16,0,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,64,128,1,float16,fp8,0,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,64,128,1,fp8,fp8,0,0.026474667092164356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,64,0,1,float16,fp8,0,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,64,0,1,fp8,fp8,0,0.026021334032217663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,64,128,1,float16,float16,0,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,64,0,1,float16,float16,0,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,64,128,1,float16,fp8,0,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,64,128,1,fp8,fp8,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,64,0,1,float16,fp8,0,0.013631999492645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,64,0,1,fp8,fp8,0,0.02272533377011617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,64,128,1,float16,float16,0,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,64,0,1,float16,float16,0,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,64,128,1,float16,fp8,0,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,64,128,1,fp8,fp8,0,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,64,0,1,float16,fp8,0,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,64,0,1,fp8,fp8,0,0.022015998760859173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,64,128,1,float16,float16,0,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,64,0,1,float16,float16,0,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,64,128,1,float16,fp8,0,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,64,128,1,fp8,fp8,0,0.022266666094462078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,64,0,1,float16,fp8,0,0.013562666873137156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,64,0,1,fp8,fp8,0,0.022282667458057404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,64,128,1,float16,float16,0,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,64,0,1,float16,float16,0,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,64,128,1,float16,fp8,0,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,64,128,1,fp8,fp8,0,0.02204799900452296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,64,0,1,float16,fp8,0,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,64,0,1,fp8,fp8,0,0.02239999920129776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,64,128,1,float16,float16,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,64,128,1,float16,fp8,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,64,0,1,float16,float16,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,64,128,1,fp8,fp8,0,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,64,0,1,float16,fp8,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,64,0,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,64,128,1,float16,float16,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,64,0,1,float16,float16,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,64,128,1,float16,fp8,0,0.012240000069141388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,64,128,1,fp8,fp8,0,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,64,128,1,fp8,fp8,0,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,64,0,1,float16,fp8,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,64,0,1,fp8,fp8,0,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,64,128,1,float16,float16,0,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,64,0,1,float16,float16,0,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,64,128,1,float16,fp8,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,64,0,1,float16,fp8,0,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,64,0,1,fp8,fp8,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,64,128,1,float16,float16,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,64,0,1,float16,float16,0,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,64,128,1,float16,fp8,0,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,64,128,1,fp8,fp8,0,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,64,0,1,float16,fp8,0,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,64,0,1,fp8,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,64,128,1,float16,float16,0,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,64,0,1,float16,float16,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,64,128,1,float16,fp8,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,64,128,1,fp8,fp8,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,64,0,1,float16,fp8,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,64,0,1,fp8,fp8,0,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,64,128,1,float16,float16,0,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,64,0,1,float16,float16,0,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,64,128,1,float16,fp8,0,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,64,128,1,fp8,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,64,0,1,float16,fp8,0,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,64,0,1,fp8,fp8,0,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,64,128,1,float16,float16,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,64,0,1,float16,float16,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,64,128,1,float16,fp8,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,64,0,1,float16,float16,0,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,64,128,1,fp8,fp8,0,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,64,0,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,64,128,1,float16,float16,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,64,128,1,float16,fp8,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,64,128,1,fp8,fp8,0,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,64,0,1,float16,fp8,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,64,0,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,64,128,1,float16,float16,0,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,64,0,1,float16,float16,0,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,64,128,1,float16,fp8,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,64,128,1,fp8,fp8,0,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,64,0,1,float16,fp8,0,0.012522666404644648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,64,0,1,fp8,fp8,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,64,128,1,float16,float16,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,64,0,1,float16,float16,0,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,64,128,1,float16,fp8,0,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,64,128,1,fp8,fp8,0,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,64,0,1,float16,fp8,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,64,0,1,fp8,fp8,0,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,64,128,1,float16,float16,0,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,64,0,1,fp8,fp8,0,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,64,0,1,float16,float16,0,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,64,128,1,float16,fp8,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,64,128,1,fp8,fp8,0,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,64,0,1,float16,fp8,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,64,128,1,float16,float16,0,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,64,0,1,float16,float16,0,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,64,128,1,float16,fp8,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,64,128,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,64,0,1,float16,fp8,0,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,64,0,1,fp8,fp8,0,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,64,128,1,float16,float16,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,64,0,1,float16,float16,0,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,64,128,1,float16,fp8,0,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,64,0,1,float16,fp8,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,64,128,1,fp8,fp8,0,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,64,0,1,fp8,fp8,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,64,128,1,float16,float16,0,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,64,0,1,float16,float16,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,64,128,1,float16,fp8,0,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,64,128,1,fp8,fp8,0,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,64,0,1,float16,fp8,0,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,64,0,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,64,128,1,float16,float16,0,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,64,0,1,float16,float16,0,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,64,128,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,64,128,1,fp8,fp8,0,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,64,0,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,64,0,1,fp8,fp8,0,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,64,128,1,float16,float16,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,64,0,1,float16,float16,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,64,128,1,float16,fp8,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,64,128,1,fp8,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,64,128,1,float16,fp8,0,0.03542399903138479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,64,0,1,float16,fp8,0,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,64,0,1,fp8,fp8,0,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,64,128,1,float16,float16,0,0.03542399903138479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,64,0,1,float16,float16,0,0.035162667433420815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,64,128,1,fp8,fp8,0,0.097653329372406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,64,0,1,float16,fp8,0,0.035232000052928925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,64,0,1,fp8,fp8,0,0.09685867031415303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,64,128,1,float16,float16,0,0.035418666899204254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,64,0,1,float16,float16,0,0.03565866748491923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,64,128,1,float16,fp8,0,0.03543466577927271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,64,128,1,fp8,fp8,0,0.09700266520182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,64,0,1,float16,fp8,0,0.03565866748491923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,64,0,1,fp8,fp8,0,0.09822400410970052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,64,128,1,float16,float16,0,0.03621866554021835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,64,0,1,float16,float16,0,0.03619733452796936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,64,128,1,float16,fp8,0,0.03624533365170161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,64,128,1,fp8,fp8,0,0.09866133332252502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,64,0,1,float16,fp8,0,0.03642133375008901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,64,0,1,fp8,fp8,0,0.09770666559537251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,64,128,1,float16,float16,0,0.024570666253566742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,64,0,1,float16,float16,0,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,64,128,1,float16,fp8,0,0.024245334168275196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,64,128,1,fp8,fp8,0,0.05875200033187866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,64,0,1,float16,fp8,0,0.02418133368094762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,64,0,1,fp8,fp8,0,0.05926933387915293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,64,128,1,float16,float16,0,0.024127999941507976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,64,0,1,fp8,fp8,0,0.058090666929880776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,64,0,1,float16,float16,0,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,64,128,1,float16,fp8,0,0.023999998966852825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,64,128,1,fp8,fp8,0,0.058933332562446594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,64,128,1,fp8,fp8,0,0.058090666929880776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,64,0,1,float16,fp8,0,0.023738667368888855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,64,128,1,float16,float16,0,0.02386133372783661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,64,0,1,float16,float16,0,0.02407466620206833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,64,128,1,float16,fp8,0,0.024506665766239166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,64,128,1,float16,fp8,0,0.024714666108290356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,64,0,1,float16,fp8,0,0.02422933280467987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,64,0,1,fp8,fp8,0,0.05866133173306783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,64,128,1,float16,float16,0,0.024495999018351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,64,128,1,float16,float16,0,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,64,0,1,float16,float16,0,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,64,128,1,fp8,fp8,0,0.059119999408721924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,64,0,1,float16,fp8,0,0.024506665766239166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,64,0,1,float16,float16,0,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,64,0,1,fp8,fp8,0,0.0602453351020813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,64,128,1,float16,fp8,0,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,64,128,1,fp8,fp8,0,0.03656533360481262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,64,0,1,float16,fp8,0,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,64,0,1,fp8,fp8,0,0.037658666570981346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,64,128,1,float16,float16,0,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,64,0,1,float16,float16,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,64,128,1,float16,fp8,0,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,64,128,1,fp8,fp8,0,0.035631999373435974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,64,0,1,float16,fp8,0,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,64,0,1,fp8,fp8,0,0.036176001032193504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,64,128,1,float16,float16,0,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,64,0,1,float16,float16,0,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,64,128,1,float16,fp8,0,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,64,128,1,fp8,fp8,0,0.03629866739114126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,64,128,1,float16,fp8,0,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,64,0,1,float16,fp8,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,64,0,1,fp8,fp8,0,0.03595733394225439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,64,0,1,fp8,fp8,0,0.036133334040641785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,64,128,1,float16,float16,0,0.016074666132529575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,64,0,1,float16,float16,0,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,64,128,1,float16,fp8,0,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,64,128,1,fp8,fp8,0,0.025648000339667004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,64,128,1,fp8,fp8,0,0.035936000446478523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,64,0,1,float16,fp8,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,64,128,1,float16,float16,0,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,64,0,1,float16,float16,0,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,64,0,1,float16,fp8,0,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,64,0,1,fp8,fp8,0,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,64,128,1,float16,float16,0,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,64,0,1,float16,float16,0,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,64,128,1,float16,fp8,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,64,128,1,fp8,fp8,0,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,64,0,1,float16,fp8,0,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,64,0,1,fp8,fp8,0,0.025050667424996693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,64,0,1,float16,fp8,0,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,64,0,1,fp8,fp8,0,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,64,128,1,float16,float16,0,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,64,0,1,float16,float16,0,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,64,128,1,float16,fp8,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,64,128,1,fp8,fp8,0,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,64,128,1,float16,float16,0,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,64,0,1,float16,float16,0,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,64,128,1,float16,fp8,0,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,64,128,1,fp8,fp8,0,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,64,0,1,float16,fp8,0,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,64,0,1,fp8,fp8,0,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,64,128,1,float16,float16,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,64,0,1,float16,float16,0,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,64,128,1,float16,fp8,0,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,64,128,1,fp8,fp8,0,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,64,0,1,float16,fp8,0,0.012389333297808966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,64,0,1,fp8,fp8,0,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,64,128,1,float16,float16,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,64,128,1,float16,float16,0,0.011946666985750198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,64,0,1,float16,float16,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,64,128,1,float16,fp8,0,0.012400000045696894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,64,128,1,fp8,fp8,0,0.021509334444999695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,64,0,1,float16,fp8,0,0.012666666259368261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,64,0,1,fp8,fp8,0,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,64,0,1,fp8,fp8,0,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,64,0,1,float16,float16,0,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,64,0,1,float16,float16,0,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,64,128,1,fp8,fp8,0,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,64,128,1,float16,fp8,0,0.012416000167528788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,64,128,1,fp8,fp8,0,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,64,128,1,float16,float16,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,64,0,1,float16,fp8,0,0.012597333639860153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,64,128,1,float16,fp8,0,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,64,128,1,float16,float16,0,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,64,128,1,float16,fp8,0,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,64,0,1,float16,fp8,0,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,64,0,1,fp8,fp8,0,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,64,128,1,float16,fp8,0,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,64,0,1,float16,float16,0,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,64,128,1,fp8,fp8,0,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,64,0,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,64,0,1,fp8,fp8,0,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,64,128,1,float16,float16,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,64,0,1,float16,float16,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,64,128,1,fp8,fp8,0,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,64,0,1,float16,fp8,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,64,0,1,fp8,fp8,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,64,128,1,float16,float16,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,64,0,1,float16,float16,0,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,64,128,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,64,128,1,fp8,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,64,0,1,float16,fp8,0,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,64,0,1,fp8,fp8,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,64,128,1,float16,float16,0,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,64,0,1,float16,float16,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,64,128,1,float16,fp8,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,64,128,1,fp8,fp8,0,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,64,0,1,float16,fp8,0,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,64,0,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,64,128,1,float16,float16,0,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,64,0,1,float16,float16,0,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,64,128,1,float16,fp8,0,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,64,0,1,float16,float16,0,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,64,128,1,fp8,fp8,0,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,64,0,1,float16,fp8,0,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,64,0,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,64,0,1,fp8,fp8,0,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,64,128,1,float16,float16,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,64,128,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,64,128,1,fp8,fp8,0,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,64,0,1,float16,fp8,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,64,128,1,float16,float16,0,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,64,0,1,float16,float16,0,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,64,128,1,float16,fp8,0,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,64,128,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,64,0,1,float16,fp8,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,64,0,1,fp8,fp8,0,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,64,128,1,float16,float16,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,64,0,1,fp8,fp8,0,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,64,0,1,float16,float16,0,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,64,128,1,float16,fp8,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,64,128,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,64,128,1,fp8,fp8,0,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,64,0,1,float16,fp8,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,64,128,1,float16,float16,0,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,64,0,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,64,128,1,float16,fp8,0,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,64,0,1,float16,fp8,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,64,0,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,64,0,1,fp8,fp8,0,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,64,128,1,float16,float16,0,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,64,0,1,float16,float16,0,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,64,128,1,float16,fp8,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,64,128,1,fp8,fp8,0,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,64,0,1,float16,fp8,0,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,64,128,1,float16,float16,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,64,0,1,float16,float16,0,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,64,128,1,float16,fp8,0,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,64,128,1,fp8,fp8,0,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,64,0,1,float16,fp8,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,64,0,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,64,0,1,fp8,fp8,0,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,64,128,1,float16,float16,0,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,64,0,1,float16,float16,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,64,128,1,float16,fp8,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,64,128,1,fp8,fp8,0,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,64,0,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,64,128,1,float16,float16,0,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,64,0,1,float16,float16,0,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,64,128,1,float16,fp8,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,64,128,1,fp8,fp8,0,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,64,0,1,float16,fp8,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,64,0,1,fp8,fp8,0,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,64,128,1,float16,float16,0,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,64,0,1,float16,float16,0,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,64,128,1,float16,fp8,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,64,128,1,fp8,fp8,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,64,0,1,float16,fp8,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,64,0,1,fp8,fp8,0,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,64,128,1,float16,float16,0,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,64,0,1,float16,float16,0,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,64,128,1,float16,fp8,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,64,128,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,64,128,1,fp8,fp8,0,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,64,0,1,float16,fp8,0,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,64,0,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,64,128,1,float16,float16,0,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,64,0,1,float16,float16,0,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,64,128,1,float16,fp8,0,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,64,0,1,float16,fp8,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,64,0,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,64,128,1,float16,float16,0,0.7503519852956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,64,128,1,float16,fp8,0,0.7415520350138346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,64,128,1,fp8,fp8,0,0.9607679843902588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,64,128,1,float16,float16,0,0.791968027750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,64,128,1,float16,fp8,0,0.7847413221995035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,64,128,1,fp8,fp8,0,1.0041440327962239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,64,128,1,float16,float16,0,0.8134720325469971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,64,0,1,float16,float16,0,4.817280133565267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,64,128,1,float16,fp8,0,0.8018026351928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,64,0,1,fp8,fp8,0,4.3547468185424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,64,0,1,float16,fp8,0,4.828175862630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,64,0,1,float16,float16,0,5.059263865152995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,64,128,1,fp8,fp8,0,1.0219840208689372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,64,128,1,float16,float16,0,0.44464532534281415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,64,0,1,fp8,fp8,0,4.390517234802246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,64,0,1,float16,fp8,0,4.866890589396159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,64,128,1,float16,fp8,0,0.43813331921895343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,64,128,1,fp8,fp8,0,0.5579093297322592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,64,0,1,float16,float16,0,4.933509190877278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,64,128,1,float16,float16,0,0.3830453157424927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,64,0,1,float16,float16,0,2.4831199645996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,64,128,1,float16,fp8,0,0.3781919876734416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,64,128,1,fp8,fp8,0,0.5034826596577963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,64,0,1,fp8,fp8,0,2.277957280476888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,64,0,1,float16,fp8,0,2.5688959757486978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,64,0,1,fp8,fp8,0,4.399146715799968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,64,128,1,float16,float16,0,0.405189315478007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,64,0,1,float16,fp8,0,5.026986757914226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,64,128,1,float16,fp8,0,0.4103093147277832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,64,0,1,float16,float16,0,2.4693759282430015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,64,128,1,fp8,fp8,0,0.5206826527913412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,64,0,1,fp8,fp8,0,2.213482697804769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,64,0,1,float16,fp8,0,2.4177759488423667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,64,128,1,float16,float16,0,0.42636799812316895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,64,0,1,float16,float16,0,2.495375951131185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,64,128,1,float16,fp8,0,0.41149334112803143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,64,128,1,fp8,fp8,0,0.5526506503423055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,64,0,1,float16,fp8,0,2.4454612731933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,64,128,1,float16,float16,0,0.2287786602973938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,64,0,1,fp8,fp8,0,2.250698725382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,64,128,1,float16,fp8,0,0.23310933510462442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,64,128,1,fp8,fp8,0,0.3017226656277974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,64,0,1,float16,float16,0,2.5913599332173667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,64,128,1,float16,float16,0,0.2118613322575887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,64,0,1,float16,float16,0,1.3126613299051921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,64,0,1,float16,fp8,0,2.4923787117004395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,64,0,1,float16,fp8,0,1.3092906475067139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,64,0,1,fp8,fp8,0,2.3105759620666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,64,128,1,float16,fp8,0,0.21719467639923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,64,0,1,fp8,fp8,0,1.1244746843973796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,64,128,1,fp8,fp8,0,0.2877066731452942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,64,0,1,float16,float16,0,1.3119680086771648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,64,128,1,float16,float16,0,0.22139199574788412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,64,128,1,float16,fp8,0,0.2192959984143575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,64,128,1,fp8,fp8,0,0.2926986614863078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,64,0,1,fp8,fp8,0,1.117466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,64,0,1,float16,fp8,0,1.294474681218465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,64,128,1,float16,float16,0,0.22447466850280762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,64,0,1,float16,float16,0,1.3039039770762126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,64,128,1,float16,fp8,0,0.2256960074106852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,64,128,1,fp8,fp8,0,0.29790399471918744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,64,0,1,fp8,fp8,0,1.1023093064626057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,64,0,1,float16,fp8,0,1.294927994410197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,64,128,1,float16,float16,0,0.15897066394488016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,64,0,1,float16,float16,0,1.3082293669382732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,64,128,1,float16,fp8,0,0.15825600425402322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,64,128,1,fp8,fp8,0,0.20014933745066324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,64,0,1,float16,fp8,0,1.27947203318278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,64,0,1,fp8,fp8,0,1.1321386496225994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,64,0,1,float16,float16,0,0.6561013460159302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,64,128,1,float16,float16,0,0.1560640037059784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,64,0,1,fp8,fp8,0,0.6122933228810629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,64,0,1,float16,fp8,0,0.6493120193481445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,64,128,1,float16,fp8,0,0.1564533313115438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,64,128,1,fp8,fp8,0,0.1962933341662089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,64,0,1,float16,float16,0,0.6574346621831259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,64,128,1,float16,float16,0,0.15876799821853638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,64,0,1,float16,fp8,0,0.6573439836502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,64,0,1,fp8,fp8,0,0.5984160105387369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,64,128,1,float16,fp8,0,0.1566933294137319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,64,128,1,fp8,fp8,0,0.1959999998410543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,64,0,1,float16,float16,0,0.648416002591451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,64,128,1,float16,float16,0,0.15842666228612265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,64,0,1,float16,fp8,0,0.6585493485132853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,64,128,1,float16,fp8,0,0.15753600001335144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,64,0,1,fp8,fp8,0,0.6097546815872192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,64,0,1,float16,float16,0,0.6480053265889486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,64,128,1,fp8,fp8,0,0.19671465953191122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,64,0,1,fp8,fp8,0,0.6058826843897501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,64,128,1,float16,float16,0,0.5734826723734537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,64,0,1,float16,fp8,0,0.6545066833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,64,128,1,float16,fp8,0,0.5600533485412598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,64,128,1,fp8,fp8,0,0.7421013514200846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,64,128,1,float16,float16,0,0.6036373376846313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,64,128,1,float16,fp8,0,0.5977919896443685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,64,128,1,fp8,fp8,0,0.759605328241984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,64,0,1,float16,float16,0,2.8801867167154946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,64,128,1,float16,float16,0,0.6201226711273193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,64,0,1,float16,fp8,0,2.8488054275512695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,64,0,1,fp8,fp8,0,2.563920021057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,64,0,1,float16,float16,0,2.8671468098958335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,64,128,1,float16,fp8,0,0.6192053159077963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,64,128,1,fp8,fp8,0,0.7801600297292074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,64,0,1,float16,fp8,0,2.866239865620931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,64,128,1,float16,float16,0,0.3404373327891032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,64,0,1,fp8,fp8,0,2.5935413042704263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,64,128,1,float16,fp8,0,0.3283146619796753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,64,128,1,fp8,fp8,0,0.4376853307088216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,64,0,1,float16,float16,0,2.883861223856608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,64,128,1,float16,float16,0,0.2911733388900757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,64,0,1,float16,float16,0,1.475935935974121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,64,0,1,fp8,fp8,0,1.305087963740031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,64,0,1,float16,fp8,0,1.4637600580851238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,64,128,1,fp8,fp8,0,0.3847893476486206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,64,128,1,float16,fp8,0,0.29134400685628253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,64,0,1,fp8,fp8,0,2.6132052739461265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,64,0,1,float16,fp8,0,2.878037452697754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,64,0,1,float16,float16,0,1.4634772936503093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,64,128,1,float16,float16,0,0.2974399924278259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,64,128,1,float16,fp8,0,0.2955519954363505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,64,128,1,fp8,fp8,0,0.39534934361775714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,64,0,1,fp8,fp8,0,1.2454986572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,64,0,1,float16,fp8,0,1.4269812901814778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,64,128,1,float16,float16,0,0.3081546624501546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,64,0,1,float16,float16,0,1.4586399396260579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,64,128,1,float16,fp8,0,0.3082080086072286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,64,128,1,fp8,fp8,0,0.4057493209838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,64,0,1,fp8,fp8,0,1.2750879923502605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,64,128,1,float16,float16,0,0.18082666397094727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,64,0,1,float16,fp8,0,1.4295679728190105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,64,128,1,float16,fp8,0,0.178330659866333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,64,0,1,float16,float16,0,1.4465866088867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,64,0,1,float16,float16,0,0.7240800062815348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,64,128,1,fp8,fp8,0,0.2389706571896871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,64,0,1,fp8,fp8,0,1.285648028055827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,64,0,1,float16,fp8,0,1.4750827153523762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,64,128,1,float16,float16,0,0.1653386652469635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,64,128,1,float16,fp8,0,0.1662346621354421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,64,0,1,fp8,fp8,0,0.6589226722717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,64,0,1,float16,fp8,0,0.7129866282145182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,64,128,1,fp8,fp8,0,0.21870400508244833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,64,0,1,float16,float16,0,0.7144052982330322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,64,128,1,float16,float16,0,0.1702079971631368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,64,128,1,float16,fp8,0,0.16589867075284323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,64,0,1,float16,fp8,0,0.718122641245524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,64,0,1,fp8,fp8,0,0.6592053174972534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,64,128,1,fp8,fp8,0,0.22242667277654013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,64,0,1,float16,float16,0,0.7178560098012289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,64,128,1,float16,float16,0,0.16967467466990152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,64,0,1,float16,fp8,0,0.7154186566670736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,64,128,1,float16,fp8,0,0.17457600434621176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,64,128,1,fp8,fp8,0,0.2270453373591105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,64,0,1,fp8,fp8,0,0.6586026748021444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,64,0,1,float16,float16,0,0.7223200003306071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,64,128,1,float16,float16,0,0.12609600027402243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,64,0,1,fp8,fp8,0,0.6552960077921549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,64,0,1,float16,fp8,0,0.7202133337656657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,64,128,1,float16,fp8,0,0.12483200430870056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,64,0,1,float16,float16,0,0.3964586655298869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,64,128,1,fp8,fp8,0,0.15947733322779337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,64,128,1,float16,float16,0,0.12174399693806966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,64,0,1,float16,fp8,0,0.39825598398844403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,64,0,1,fp8,fp8,0,0.3694186607996623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,64,128,1,float16,fp8,0,0.12187199791272481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,64,0,1,float16,float16,0,0.3961600065231323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,64,128,1,float16,float16,0,0.1239359974861145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,64,128,1,fp8,fp8,0,0.15500266353289285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,64,128,1,float16,fp8,0,0.12292266885439555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,64,0,1,float16,fp8,0,0.3928320010503133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,64,0,1,fp8,fp8,0,0.3641173442204793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,64,0,1,float16,float16,0,0.39477332433064777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,64,128,1,fp8,fp8,0,0.1570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,64,0,1,float16,fp8,0,0.39740800857543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,64,128,1,float16,float16,0,0.12346667051315308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,64,0,1,fp8,fp8,0,0.3661280075709025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,64,128,1,float16,fp8,0,0.1239520013332367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,64,0,1,float16,float16,0,0.3944000005722046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,64,128,1,fp8,fp8,0,0.15668800473213196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,64,0,1,float16,fp8,0,0.39847465356191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,64,0,1,fp8,fp8,0,0.3725759983062744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,64,128,1,float16,float16,0,0.46952001253763836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,64,128,1,float16,fp8,0,0.45953599611918133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,64,128,1,fp8,fp8,0,0.6145386695861816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,64,128,1,float16,float16,0,0.5063039859135946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,64,128,1,float16,fp8,0,0.5030719836552938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,64,128,1,fp8,fp8,0,0.6436426639556885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,64,0,1,float16,float16,0,1.9980479876200359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,64,0,1,float16,fp8,0,2.0119360287984214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,64,0,1,fp8,fp8,0,1.843605359395345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,64,0,1,float16,float16,0,2.0276692708333335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,64,128,1,float16,float16,0,0.5199733177820841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,64,128,1,float16,fp8,0,0.51528000831604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,64,0,1,float16,fp8,0,2.0407840410868325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,64,128,1,fp8,fp8,0,0.6538559993108114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,64,128,1,float16,float16,0,0.2828960021336873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,64,0,1,fp8,fp8,0,1.8935252825419109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,64,128,1,float16,fp8,0,0.2756693363189697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,64,128,1,fp8,fp8,0,0.3619680007298787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,64,0,1,float16,float16,0,2.03984006245931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,64,0,1,float16,fp8,0,2.0674079259236655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,64,128,1,float16,float16,0,0.24565333127975464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,64,0,1,float16,float16,0,1.0786933104197185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,64,0,1,float16,fp8,0,1.0738506317138672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,64,0,1,fp8,fp8,0,1.9037760098775227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,64,0,1,fp8,fp8,0,0.9398453235626221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,64,0,1,float16,float16,0,1.046730677286784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,64,128,1,float16,fp8,0,0.24582932392756143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,64,128,1,fp8,fp8,0,0.3320106665293376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,64,128,1,float16,float16,0,0.2539093295733134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,64,128,1,float16,fp8,0,0.24767466386159262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,64,0,1,float16,fp8,0,1.0315679709116619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,64,0,1,fp8,fp8,0,0.9065759976704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,64,128,1,fp8,fp8,0,0.33766400814056396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,64,128,1,float16,float16,0,0.26146133740743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,64,0,1,float16,float16,0,1.0393919944763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,64,0,1,float16,fp8,0,1.054426670074463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,64,0,1,fp8,fp8,0,0.9307466348012289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,64,128,1,float16,fp8,0,0.25752532482147217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,64,128,1,fp8,fp8,0,0.34490664800008136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,64,128,1,float16,float16,0,0.15372266372044882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,64,0,1,float16,float16,0,1.0437066555023193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,64,128,1,float16,fp8,0,0.1562879979610443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,64,0,1,float16,float16,0,0.5204213460286459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,64,128,1,fp8,fp8,0,0.2071733276049296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,64,0,1,fp8,fp8,0,0.9251413345336914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,64,0,1,float16,fp8,0,1.0555413564046223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,64,128,1,float16,float16,0,0.1411733329296112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,64,0,1,float16,fp8,0,0.5214666525522867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,64,0,1,fp8,fp8,0,0.48640533288319904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,64,128,1,float16,fp8,0,0.1413279970486959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,64,128,1,fp8,fp8,0,0.19139732917149863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,64,0,1,float16,float16,0,0.5113706588745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,64,0,1,float16,fp8,0,0.5102666616439819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,64,128,1,float16,float16,0,0.14264532923698425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,64,0,1,fp8,fp8,0,0.4774186611175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,64,128,1,float16,fp8,0,0.14359466234842935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,64,0,1,float16,float16,0,0.518234650293986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,64,128,1,fp8,fp8,0,0.19293866554896036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,64,128,1,float16,float16,0,0.14597333470980325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,64,0,1,float16,fp8,0,0.5124746561050415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,64,0,1,fp8,fp8,0,0.4748053153355916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,64,128,1,float16,fp8,0,0.14724266529083252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,64,0,1,float16,float16,0,0.5142133235931396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,64,128,1,fp8,fp8,0,0.20068800449371338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,64,128,1,float16,float16,0,0.10704533259073894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,64,0,1,float16,fp8,0,0.5156480073928833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,64,0,1,float16,fp8,0,0.29445866743723553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,64,0,1,fp8,fp8,0,0.47842665513356525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,64,0,1,float16,float16,0,0.29391467571258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,64,128,1,float16,fp8,0,0.10738133390744527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,64,128,1,fp8,fp8,0,0.1366986632347107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,64,0,1,fp8,fp8,0,0.27266132831573486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,64,128,1,float16,float16,0,0.10317333539326985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,64,128,1,float16,fp8,0,0.10268266995747884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,64,0,1,float16,float16,0,0.292303999265035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,64,128,1,fp8,fp8,0,0.12505066394805908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,64,0,1,float16,fp8,0,0.2934826612472534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,64,0,1,fp8,fp8,0,0.2699306607246399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,64,128,1,float16,float16,0,0.1046453317006429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,64,128,1,float16,fp8,0,0.10421866178512573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,64,0,1,float16,fp8,0,0.2917120059331258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,64,0,1,float16,float16,0,0.2924853364626567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,64,128,1,fp8,fp8,0,0.13306666413942972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,64,128,1,float16,float16,0,0.10616532961527507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,64,128,1,fp8,fp8,0,0.13528533776601157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,64,0,1,fp8,fp8,0,0.2707306742668152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,64,0,1,float16,float16,0,0.29362134138743085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,64,128,1,float16,fp8,0,0.10594666997591655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,64,0,1,float16,fp8,0,0.2932373285293579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,64,0,1,fp8,fp8,0,0.2733280062675476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,64,128,1,float16,float16,0,0.7433706919352213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,64,128,1,float16,fp8,0,0.7344533602396647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,64,128,1,fp8,fp8,0,0.951573371887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,64,128,1,float16,float16,0,0.788325309753418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,64,128,1,float16,fp8,0,0.7763733069101969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,64,0,1,float16,float16,0,2.693301200866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,64,128,1,fp8,fp8,0,0.9933386643727621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,64,0,1,fp8,fp8,0,2.4221386909484863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,64,0,1,float16,fp8,0,2.643237272898356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,64,128,1,float16,float16,0,0.8065439860026041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,64,0,1,float16,float16,0,2.715872128804525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,64,128,1,float16,fp8,0,0.7954826354980469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,64,0,1,fp8,fp8,0,2.468191941579183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,64,128,1,float16,float16,0,0.44050133228302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,64,128,1,fp8,fp8,0,1.0155466397603352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,64,0,1,float16,fp8,0,2.704986572265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,64,128,1,float16,fp8,0,0.4323466618855794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,64,128,1,fp8,fp8,0,0.5546293258666992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,64,0,1,float16,float16,0,2.7826719284057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,64,0,1,float16,float16,0,1.3944160143534343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,64,128,1,float16,float16,0,0.3713279962539673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,64,0,1,float16,fp8,0,2.7161385218302407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,64,0,1,fp8,fp8,0,2.485962708791097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,64,128,1,float16,fp8,0,0.3779093424479167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,64,0,1,float16,fp8,0,1.3992160161336262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,64,128,1,fp8,fp8,0,0.4984426498413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,64,0,1,fp8,fp8,0,1.300986687342326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,64,0,1,float16,float16,0,1.3282933235168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,64,128,1,float16,float16,0,0.3980853160222371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,64,128,1,float16,fp8,0,0.39845867951711017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,64,0,1,float16,fp8,0,1.3284586270650227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,64,128,1,fp8,fp8,0,0.522490660349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,64,0,1,fp8,fp8,0,1.2378826936086018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,64,128,1,float16,float16,0,0.41174932320912677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,64,0,1,float16,float16,0,1.3574825922648113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,64,128,1,float16,fp8,0,0.4065706729888916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,64,128,1,fp8,fp8,0,0.5352693398793539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,64,128,1,float16,float16,0,0.22499734163284302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,64,0,1,fp8,fp8,0,1.2864373524983723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,64,0,1,float16,fp8,0,1.374176025390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,64,0,1,float16,float16,0,1.3835093180338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,64,128,1,float16,fp8,0,0.22316267093022665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,64,128,1,fp8,fp8,0,0.2940160036087036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,64,0,1,fp8,fp8,0,1.2746079762776692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,64,0,1,float16,fp8,0,1.3762987454732258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,64,0,1,float16,float16,0,0.7302186489105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,64,128,1,float16,float16,0,0.20222934087117514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,64,0,1,float16,fp8,0,0.7321759859720866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,64,128,1,float16,fp8,0,0.19740800062815347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,64,0,1,fp8,fp8,0,0.6449386676152548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,64,128,1,fp8,fp8,0,0.26505066951115924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,64,0,1,float16,float16,0,0.7083413600921631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,64,128,1,float16,float16,0,0.2041013240814209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,64,0,1,float16,fp8,0,0.7106453577677408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,64,0,1,fp8,fp8,0,0.6167093515396118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,64,128,1,float16,fp8,0,0.2027413249015808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,64,128,1,fp8,fp8,0,0.27721067269643146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,64,0,1,float16,float16,0,0.7161386807759603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,64,128,1,float16,float16,0,0.20985599358876547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,64,0,1,float16,fp8,0,0.7098293304443359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,64,0,1,fp8,fp8,0,0.6217173337936401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,64,128,1,float16,fp8,0,0.21008533239364624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,64,128,1,fp8,fp8,0,0.28389867146809894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,64,0,1,float16,float16,0,0.721455971399943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,64,128,1,float16,float16,0,0.129013329744339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,64,0,1,float16,float16,0,0.35783998171488446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,64,0,1,float16,fp8,0,0.7207466761271158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,64,0,1,float16,fp8,0,0.3612053394317627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,64,128,1,float16,fp8,0,0.12973333398501077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,64,0,1,fp8,fp8,0,0.6226346492767334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,64,128,1,fp8,fp8,0,0.17546667655309042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,64,128,1,float16,float16,0,0.1190826694170634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,64,0,1,fp8,fp8,0,0.33850133419036865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,64,128,1,float16,fp8,0,0.11906133095423381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,64,128,1,fp8,fp8,0,0.15829867124557495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,64,0,1,float16,float16,0,0.35204799969991046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,64,128,1,float16,float16,0,0.11969600121180217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,64,0,1,float16,fp8,0,0.34487998485565186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,64,0,1,fp8,fp8,0,0.3266613284746806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,64,128,1,float16,fp8,0,0.12025066216786702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,64,128,1,fp8,fp8,0,0.16040533781051636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,64,0,1,float16,float16,0,0.35050666332244873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,64,0,1,float16,fp8,0,0.3481920162836711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,64,128,1,float16,float16,0,0.12110933661460876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,64,128,1,float16,fp8,0,0.12332800030708313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,64,0,1,fp8,fp8,0,0.32818132638931274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,64,0,1,float16,fp8,0,0.3504319985707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,64,0,1,float16,float16,0,0.353434681892395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,64,128,1,fp8,fp8,0,0.1639253298441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,64,128,1,float16,float16,0,0.09038399656613667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,64,128,1,float16,fp8,0,0.0906773308912913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,64,0,1,fp8,fp8,0,0.3296533425649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,64,0,1,float16,float16,0,0.20574400822321573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,64,128,1,fp8,fp8,0,0.11669333775838216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,64,0,1,float16,fp8,0,0.202949325243632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,64,128,1,float16,float16,0,0.08716799815495808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,64,0,1,fp8,fp8,0,0.19128000736236572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,64,0,1,float16,float16,0,0.2002453406651815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,64,128,1,float16,fp8,0,0.08803199728329976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,64,128,1,fp8,fp8,0,0.10541333754857381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,64,0,1,float16,fp8,0,0.20105600357055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,64,0,1,fp8,fp8,0,0.18864534298578897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,64,128,1,float16,float16,0,0.08750933408737183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,64,0,1,float16,float16,0,0.1999359925587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,64,128,1,float16,fp8,0,0.0876533289750417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,64,128,1,fp8,fp8,0,0.10533333818117778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,64,0,1,float16,fp8,0,0.20232532421747842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,64,0,1,fp8,fp8,0,0.18903466065724692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,64,128,1,float16,float16,0,0.0895306666692098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,64,128,1,float16,fp8,0,0.08966400225957234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,64,0,1,float16,float16,0,0.20164267222086588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,64,128,1,fp8,fp8,0,0.10677867134412129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,64,0,1,fp8,fp8,0,0.19129067659378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,64,0,1,float16,fp8,0,0.20160533984502158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,64,128,1,float16,float16,0,0.5579253435134888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,64,128,1,float16,fp8,0,0.5512746572494507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,64,128,1,fp8,fp8,0,0.722437302271525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,64,128,1,float16,float16,0,0.5969760020573934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,64,0,1,float16,float16,0,1.5933547019958496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,64,128,1,float16,fp8,0,0.5886720021565756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,64,0,1,fp8,fp8,0,1.4677386283874512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,64,128,1,fp8,fp8,0,0.7546772956848145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,64,128,1,float16,float16,0,0.6157973210016886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,64,0,1,float16,fp8,0,1.569994608561198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,64,0,1,float16,float16,0,1.6172480583190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,64,128,1,float16,fp8,0,0.6023306846618652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,64,0,1,fp8,fp8,0,1.5075573921203613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,64,0,1,float16,fp8,0,1.6095306078592937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,64,128,1,fp8,fp8,0,0.7726453145345052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,64,128,1,float16,float16,0,0.3319200078646342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,64,128,1,float16,fp8,0,0.32531734307607013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,64,0,1,float16,float16,0,1.6344480514526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,64,128,1,fp8,fp8,0,0.4266773462295532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,64,0,1,float16,float16,0,0.8535253206888834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,64,0,1,float16,fp8,0,1.6264692942301433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,64,128,1,float16,float16,0,0.2829226652781169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,64,0,1,float16,fp8,0,0.845957358678182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,64,128,1,float16,fp8,0,0.27662932872772217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,64,0,1,fp8,fp8,0,1.5151146252950032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,64,0,1,fp8,fp8,0,0.7703519662221273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,64,128,1,fp8,fp8,0,0.37218133608500165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,64,0,1,float16,float16,0,0.8054186503092448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,64,128,1,float16,float16,0,0.2914453347524007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,64,128,1,float16,fp8,0,0.28641066948572796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,64,0,1,fp8,fp8,0,0.7177813053131104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,64,0,1,float16,fp8,0,0.8041226863861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,64,128,1,fp8,fp8,0,0.3895839850107829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,64,0,1,float16,float16,0,0.8136373360951742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,64,128,1,float16,float16,0,0.30301332473754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,64,0,1,float16,fp8,0,0.8091839949289957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,64,128,1,float16,fp8,0,0.2965173323949178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,64,0,1,fp8,fp8,0,0.7336426575978597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,64,128,1,fp8,fp8,0,0.40274667739868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,64,0,1,float16,float16,0,0.8247360388437907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,64,128,1,float16,float16,0,0.17508800824483237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,64,128,1,float16,fp8,0,0.171450674533844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,64,0,1,float16,float16,0,0.4152853488922119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,64,128,1,fp8,fp8,0,0.22906132539113364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,64,0,1,fp8,fp8,0,0.7483146985371908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,64,0,1,float16,fp8,0,0.8298986752827963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,64,128,1,float16,float16,0,0.1576586663722992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,64,0,1,float16,fp8,0,0.42345066865285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,64,0,1,fp8,fp8,0,0.3899093468983968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,64,128,1,float16,fp8,0,0.15110400319099426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,64,0,1,float16,float16,0,0.3992533286412557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,64,128,1,fp8,fp8,0,0.2076479991277059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,64,0,1,float16,fp8,0,0.4018933375676473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,64,128,1,float16,float16,0,0.16078933080037436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,64,0,1,fp8,fp8,0,0.3779413302739461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,64,128,1,float16,fp8,0,0.157258669535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,64,0,1,float16,float16,0,0.40596266587575275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,64,128,1,fp8,fp8,0,0.21370132764180502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,64,128,1,float16,float16,0,0.16581867138544717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,64,0,1,float16,fp8,0,0.4057759841283162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,64,0,1,fp8,fp8,0,0.3821013371149699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,64,128,1,float16,fp8,0,0.16377600034077963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,64,0,1,float16,float16,0,0.4103519916534424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,64,128,1,fp8,fp8,0,0.21901333332061768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,64,0,1,float16,fp8,0,0.40529600779215497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,64,128,1,float16,fp8,0,0.10289600491523743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,64,128,1,float16,float16,0,0.10250133275985718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,64,0,1,fp8,fp8,0,0.3859626849492391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,64,0,1,fp8,fp8,0,0.21109867095947266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,64,0,1,float16,float16,0,0.2209706703821818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,64,128,1,fp8,fp8,0,0.13736533125241598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,64,0,1,float16,fp8,0,0.22422399123509726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,64,128,1,float16,float16,0,0.09236799677213033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,64,0,1,float16,float16,0,0.21636799971262613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,64,0,1,fp8,fp8,0,0.19996267557144165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,64,128,1,float16,fp8,0,0.0934933324654897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,64,128,1,fp8,fp8,0,0.12532266974449158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,64,0,1,float16,fp8,0,0.21405333280563354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,64,128,1,float16,float16,0,0.09462400277455647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,64,0,1,float16,float16,0,0.21557333072026572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,64,0,1,float16,fp8,0,0.21794666846593222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,64,128,1,float16,fp8,0,0.0943946639696757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,64,128,1,float16,float16,0,0.09659733374913533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,64,128,1,fp8,fp8,0,0.12769599755605063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,64,0,1,fp8,fp8,0,0.2021920084953308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,64,0,1,float16,float16,0,0.2178186575571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,64,0,1,float16,fp8,0,0.2181439995765686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,64,128,1,float16,fp8,0,0.09683733185132344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,64,0,1,float16,float16,0,0.13570132851600647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,64,128,1,fp8,fp8,0,0.1338933308919271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,64,128,1,float16,float16,0,0.07306666672229767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,64,0,1,fp8,fp8,0,0.20694933334986368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,64,128,1,float16,fp8,0,0.07155733307202657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,64,128,1,fp8,fp8,0,0.09409067034721375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,64,0,1,float16,fp8,0,0.1355946660041809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,64,0,1,fp8,fp8,0,0.1307253340880076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,64,128,1,float16,float16,0,0.07017066578070323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,64,0,1,float16,float16,0,0.1360213359196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,64,128,1,float16,fp8,0,0.069541335105896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,64,128,1,fp8,fp8,0,0.08716266353925069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,64,0,1,float16,fp8,0,0.1362773378690084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,64,0,1,fp8,fp8,0,0.12773866454760233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,64,128,1,float16,float16,0,0.07090666890144348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,64,0,1,float16,float16,0,0.1365120013554891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,64,128,1,float16,fp8,0,0.06994133194287618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,64,128,1,fp8,fp8,0,0.08714666962623596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,64,0,1,float16,fp8,0,0.13613866766293845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,64,0,1,fp8,fp8,0,0.1284266710281372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,64,128,1,float16,float16,0,0.06924266616503398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,64,0,1,float16,float16,0,0.1353493332862854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,64,128,1,float16,fp8,0,0.07005333403746287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,64,128,1,fp8,fp8,0,0.08695466319719951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,64,0,1,float16,fp8,0,0.1357493301232656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,64,0,1,fp8,fp8,0,0.12873599926630655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,64,128,1,float16,fp8,0,0.734773317972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,64,128,1,float16,float16,0,0.7465813159942627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,64,128,1,fp8,fp8,0,0.9450080394744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,64,128,1,float16,float16,0,0.7833279768625895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,64,0,1,float16,float16,0,1.568661371866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,64,0,1,float16,fp8,0,1.5567253430684407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,64,0,1,fp8,fp8,0,1.4542880058288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,64,128,1,float16,fp8,0,0.7750826676686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,64,0,1,float16,float16,0,1.6085333824157715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,64,128,1,fp8,fp8,0,0.9897920290629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,64,128,1,float16,float16,0,0.8085119724273682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,64,128,1,float16,fp8,0,0.7962933381398519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,64,0,1,float16,fp8,0,1.6142880121866863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,64,0,1,fp8,fp8,0,1.493285338083903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,64,128,1,float16,float16,0,0.437173326810201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,64,128,1,fp8,fp8,0,1.0111040274302165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,64,0,1,float16,float16,0,1.6383840243021648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,64,128,1,float16,fp8,0,0.4317706823348999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,64,0,1,float16,fp8,0,1.6219040552775066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,64,128,1,fp8,fp8,0,0.55240531762441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,64,0,1,float16,float16,0,0.8567787011464437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,64,0,1,fp8,fp8,0,1.5186667442321777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,64,0,1,fp8,fp8,0,0.8056853612263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,64,0,1,float16,fp8,0,0.8467040061950684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,64,128,1,float16,float16,0,0.36670398712158203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,64,128,1,float16,fp8,0,0.3621866703033447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,64,0,1,float16,float16,0,0.7963306903839111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,64,128,1,fp8,fp8,0,0.48476799329121906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,64,128,1,float16,float16,0,0.3959360122680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,64,0,1,float16,fp8,0,0.7803893089294434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,64,0,1,fp8,fp8,0,0.7408426602681478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,64,128,1,float16,fp8,0,0.3909226655960083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,64,0,1,float16,float16,0,0.821295976638794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,64,128,1,fp8,fp8,0,0.5102826754252116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,64,0,1,float16,fp8,0,0.8166560331980387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,64,128,1,float16,float16,0,0.4081759850184123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,64,0,1,fp8,fp8,0,0.764959971110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,64,128,1,float16,fp8,0,0.40294400850931805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,64,128,1,fp8,fp8,0,0.5227946837743124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,64,0,1,float16,float16,0,0.8361866474151611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,64,128,1,float16,float16,0,0.22168533007303873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,64,0,1,float16,fp8,0,0.8277866840362549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,64,0,1,float16,float16,0,0.4383093516031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,64,128,1,float16,fp8,0,0.21786133448282877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,64,0,1,fp8,fp8,0,0.7860373655954996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,64,128,1,fp8,fp8,0,0.28706665833791095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,64,0,1,float16,fp8,0,0.43995734055836994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,64,128,1,float16,fp8,0,0.19038933515548706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,64,0,1,fp8,fp8,0,0.3946346839269002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,64,128,1,fp8,fp8,0,0.2614826758702596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,64,128,1,float16,float16,0,0.1957333286603292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,64,0,1,float16,float16,0,0.4131786823272705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,64,0,1,float16,fp8,0,0.4148586591084798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,64,128,1,float16,fp8,0,0.1966666579246521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,64,128,1,float16,float16,0,0.1985493302345276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,64,128,1,fp8,fp8,0,0.2682346701622009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,64,0,1,fp8,fp8,0,0.37112534046173096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,64,0,1,float16,float16,0,0.41832534472147626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,64,0,1,float16,fp8,0,0.4140479962031047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,64,0,1,fp8,fp8,0,0.3773706754048665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,64,128,1,float16,float16,0,0.20563733577728271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,64,128,1,float16,fp8,0,0.20532800753911337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,64,0,1,float16,float16,0,0.42501866817474365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,64,128,1,fp8,fp8,0,0.275434672832489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,64,128,1,float16,float16,0,0.12401599685351054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,64,0,1,float16,fp8,0,0.41967467466990155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,64,0,1,fp8,fp8,0,0.3793333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,64,0,1,float16,float16,0,0.21846934159596762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,64,128,1,float16,fp8,0,0.12204800049463908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,64,128,1,fp8,fp8,0,0.16342399517695108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,64,0,1,float16,fp8,0,0.22192533810933432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,64,128,1,float16,float16,0,0.10902933279673259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,64,128,1,float16,fp8,0,0.10790399710337321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,64,128,1,fp8,fp8,0,0.15059733390808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,64,0,1,fp8,fp8,0,0.20928533871968588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,64,0,1,float16,float16,0,0.20692267020543417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,64,0,1,float16,fp8,0,0.20625599225362143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,64,0,1,fp8,fp8,0,0.1981226603190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,64,128,1,float16,float16,0,0.11080533266067505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,64,128,1,float16,fp8,0,0.11033067107200623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,64,0,1,float16,float16,0,0.2092533310254415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,64,128,1,fp8,fp8,0,0.1530186633268992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,64,0,1,float16,fp8,0,0.2076479991277059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,64,0,1,fp8,fp8,0,0.1981119910875956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,64,128,1,float16,float16,0,0.11601066589355469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,64,0,1,float16,float16,0,0.21146132548650107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,64,128,1,float16,fp8,0,0.11526399850845337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,64,128,1,fp8,fp8,0,0.15716800093650818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,64,0,1,float16,fp8,0,0.21130667130152384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,64,0,1,fp8,fp8,0,0.20523200432459512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,64,128,1,float16,float16,0,0.07348800202210744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,64,0,1,float16,float16,0,0.12072533369064331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,64,128,1,float16,fp8,0,0.07408533493677776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,64,128,1,fp8,fp8,0,0.10326932867368062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,64,0,1,float16,fp8,0,0.1207360029220581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,64,0,1,fp8,fp8,0,0.11758933464686076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,64,128,1,float16,float16,0,0.06776000062624614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,64,0,1,float16,float16,0,0.11691199739774068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,64,128,1,float16,fp8,0,0.06832533578077953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,64,128,1,fp8,fp8,0,0.08563199639320374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,64,0,1,float16,fp8,0,0.11546132961908977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,64,0,1,fp8,fp8,0,0.10839466253916423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,64,128,1,float16,float16,0,0.06867200136184692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,64,0,1,float16,float16,0,0.11602666974067688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,64,128,1,float16,fp8,0,0.06789866586526234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,64,128,1,fp8,fp8,0,0.08674666285514832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,64,0,1,float16,fp8,0,0.11545067032178243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,64,0,1,fp8,fp8,0,0.11091199517250061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,64,128,1,float16,float16,0,0.07121066749095917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,64,128,1,float16,float16,0,0.05406400064627329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,64,0,1,float16,float16,0,0.1181119978427887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,64,128,1,float16,fp8,0,0.07081066568692525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,64,128,1,fp8,fp8,0,0.09664533535639445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,64,0,1,float16,fp8,0,0.11580800016721089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,64,0,1,fp8,fp8,0,0.11289067069689433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,64,0,1,float16,float16,0,0.07624533275763194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,64,128,1,float16,fp8,0,0.05394133428732554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,64,128,1,fp8,fp8,0,0.06449066599210103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,64,0,1,float16,fp8,0,0.07615466912587483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,64,0,1,fp8,fp8,0,0.07187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,64,128,1,float16,float16,0,0.05400000015894572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,64,0,1,float16,float16,0,0.07518933216730754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,64,128,1,float16,fp8,0,0.053717335065205894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,64,0,1,float16,fp8,0,0.07543999950091045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,64,128,1,fp8,fp8,0,0.06401066482067108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,64,128,1,float16,float16,0,0.054048001766204834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,64,0,1,fp8,fp8,0,0.07262399792671204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,64,128,1,float16,fp8,0,0.05312533179918925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,64,128,1,fp8,fp8,0,0.0643093337615331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,64,0,1,float16,float16,0,0.07542933523654938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,64,0,1,float16,fp8,0,0.07628799974918365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,64,0,1,fp8,fp8,0,0.07235200206438701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,64,128,1,float16,float16,0,0.05357333521048228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,64,0,1,float16,float16,0,0.07597333192825317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,64,128,1,float16,fp8,0,0.05418133238951365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,64,128,1,fp8,fp8,0,0.06410133341948192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,64,0,1,float16,fp8,0,0.07580266892910004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,64,0,1,fp8,fp8,0,0.07318933308124542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,64,128,1,float16,float16,0,0.5489279826482137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,64,128,1,float16,fp8,0,0.543941338857015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,64,0,1,float16,float16,0,0.9746293226877848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,64,128,1,fp8,fp8,0,0.7003466288248698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,64,128,1,float16,float16,0,0.5777973333994547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,64,0,1,float16,fp8,0,0.9674773216247559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,64,0,1,fp8,fp8,0,0.9102559884389242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,64,128,1,float16,fp8,0,0.5682026545206705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,64,0,1,float16,float16,0,0.9973226388295492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,64,128,1,fp8,fp8,0,0.7240266799926758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,64,128,1,float16,float16,0,0.5980480114618937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,64,0,1,fp8,fp8,0,0.9262080192565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,64,0,1,float16,fp8,0,0.9927573204040527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,64,128,1,float16,fp8,0,0.5872960090637207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,64,0,1,float16,float16,0,1.0335893630981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,64,128,1,fp8,fp8,0,0.7519946893056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,64,128,1,float16,float16,0,0.32901867230733234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,64,0,1,float16,fp8,0,1.012346665064494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,64,0,1,float16,float16,0,0.5453493197758993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,64,0,1,fp8,fp8,0,0.9491573174794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,64,128,1,float16,fp8,0,0.32123200098673504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,64,128,1,fp8,fp8,0,0.418997327486674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,64,0,1,float16,fp8,0,0.5369813442230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,64,0,1,fp8,fp8,0,0.5057066679000854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,64,128,1,float16,float16,0,0.27720532814661664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,64,128,1,float16,fp8,0,0.2725333372751872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,64,0,1,float16,float16,0,0.49772799015045166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,64,128,1,fp8,fp8,0,0.3681866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,64,0,1,float16,fp8,0,0.49675734837849933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,64,0,1,fp8,fp8,0,0.4493120114008586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,64,128,1,float16,float16,0,0.2914346655209859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,64,128,1,float16,fp8,0,0.28253332773844403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,64,0,1,float16,float16,0,0.5123999913533529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,64,128,1,fp8,fp8,0,0.38076265652974445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,64,0,1,float16,fp8,0,0.5015360116958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,64,128,1,float16,float16,0,0.30084800720214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,64,0,1,fp8,fp8,0,0.466597318649292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,64,128,1,float16,fp8,0,0.2948746681213379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,64,128,1,float16,float16,0,0.17012266318003336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,64,0,1,float16,float16,0,0.5248639980951945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,64,0,1,float16,fp8,0,0.5124853452046713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,64,128,1,fp8,fp8,0,0.3979146480560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,64,0,1,float16,float16,0,0.27432000637054443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,64,0,1,fp8,fp8,0,0.47597865263621014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,64,0,1,fp8,fp8,0,0.2550719976425171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,64,128,1,fp8,fp8,0,0.22167466084162393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,64,128,1,float16,fp8,0,0.16916799545288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,64,0,1,float16,fp8,0,0.27038933833440143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,64,0,1,float16,fp8,0,0.2467893362045288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,64,128,1,float16,float16,0,0.15221333503723145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,64,0,1,float16,float16,0,0.2534293333689372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,64,128,1,float16,fp8,0,0.14940266807874045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,64,128,1,fp8,fp8,0,0.20272533098856607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,64,0,1,fp8,fp8,0,0.23553067445755005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,64,128,1,float16,float16,0,0.15312000115712485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,64,128,1,float16,fp8,0,0.1520746648311615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,64,0,1,float16,float16,0,0.2496053377787272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,64,128,1,fp8,fp8,0,0.21207465728123984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,64,0,1,float16,fp8,0,0.2541919946670532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,64,128,1,float16,float16,0,0.15848533312479654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,64,0,1,fp8,fp8,0,0.2400746742884318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,64,128,1,float16,fp8,0,0.15784533818562826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,64,0,1,float16,float16,0,0.25443732738494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,64,128,1,fp8,fp8,0,0.21434666713078818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,64,0,1,float16,fp8,0,0.25380265712738037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,64,128,1,float16,float16,0,0.09724266330401103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,64,0,1,fp8,fp8,0,0.2461493412653605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,64,0,1,float16,float16,0,0.14408000310262045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,64,128,1,float16,fp8,0,0.097680002450943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,64,128,1,fp8,fp8,0,0.13111467162768045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,64,0,1,float16,fp8,0,0.14352533221244812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,64,0,1,fp8,fp8,0,0.1397813359896342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,64,128,1,float16,float16,0,0.08627200126647949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,64,0,1,float16,float16,0,0.13426666458447775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,64,128,1,float16,fp8,0,0.08547733227411906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,64,0,1,float16,fp8,0,0.13366400202115378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,64,128,1,fp8,fp8,0,0.12085333466529846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,64,0,1,fp8,fp8,0,0.12713066736857095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,64,128,1,float16,float16,0,0.08833066622416179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,64,0,1,float16,float16,0,0.13518933455149332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,64,128,1,float16,fp8,0,0.08726933598518372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,64,128,1,fp8,fp8,0,0.12275200088818868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,64,0,1,fp8,fp8,0,0.1311253309249878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,64,0,1,float16,fp8,0,0.13541866342226663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,64,128,1,float16,float16,0,0.09088533123334248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,64,0,1,float16,float16,0,0.13874133427937826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,64,128,1,float16,fp8,0,0.09102400143941243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,64,128,1,fp8,fp8,0,0.1262506643931071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,64,128,1,float16,float16,0,0.05813866853713989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,64,0,1,float16,fp8,0,0.13740799824396768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,64,0,1,fp8,fp8,0,0.1347040037314097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,64,0,1,float16,float16,0,0.0831520011027654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,64,128,1,float16,fp8,0,0.05830933153629303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,64,128,1,fp8,fp8,0,0.08363200227419536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,64,0,1,float16,fp8,0,0.0836853285630544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,64,0,1,fp8,fp8,0,0.08084799846013387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,64,128,1,float16,float16,0,0.05482133229573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,64,0,1,float16,float16,0,0.07920533418655396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,64,128,1,float16,fp8,0,0.05446400245030721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,64,128,1,fp8,fp8,0,0.07201600074768066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,64,0,1,fp8,fp8,0,0.07555200159549713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,64,0,1,float16,fp8,0,0.0798773318529129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,64,128,1,float16,float16,0,0.05498133103052775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,64,0,1,float16,float16,0,0.07937600215276082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,64,128,1,float16,fp8,0,0.05467733244101206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,64,128,1,fp8,fp8,0,0.06941333413124084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,64,0,1,float16,fp8,0,0.07985599835713704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,64,128,1,float16,float16,0,0.055493334929148354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,64,0,1,fp8,fp8,0,0.07652799785137177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,64,0,1,float16,float16,0,0.08105066418647766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,64,0,1,fp8,fp8,0,0.07736533383528392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,64,128,1,float16,fp8,0,0.055061335364977516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,64,128,1,fp8,fp8,0,0.07363200187683105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,64,0,1,float16,fp8,0,0.08045333127180736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,64,128,1,float16,float16,0,0.04800533254941305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,64,0,1,float16,float16,0,0.05760000149408976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,64,128,1,float16,fp8,0,0.048810665806134544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,64,128,1,fp8,fp8,0,0.05036800106366476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,64,128,1,float16,fp8,0,0.04573333263397217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,64,0,1,float16,fp8,0,0.05855466425418854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,64,0,1,fp8,fp8,0,0.05596266686916351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,64,128,1,float16,float16,0,0.044853334625562034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,64,0,1,float16,float16,0,0.05719999969005585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,64,128,1,fp8,fp8,0,0.0498879998922348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,64,0,1,float16,fp8,0,0.05780800183614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,64,0,1,fp8,fp8,0,0.05603733162085215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,64,128,1,float16,float16,0,0.04711999992529551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,64,0,1,float16,float16,0,0.05699199934800466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,64,128,1,float16,float16,0,0.04540266593297323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,64,128,1,float16,fp8,0,0.04761599997679392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,64,128,1,fp8,fp8,0,0.05011733373006185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,64,0,1,float16,fp8,0,0.05718400080998739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,64,0,1,fp8,fp8,0,0.05637866755326589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,64,0,1,fp8,fp8,0,0.05598400036493937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,64,0,1,float16,float16,0,0.057477335135142006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,64,128,1,float16,fp8,0,0.04774933556715647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,64,128,1,fp8,fp8,0,0.05055999755859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,64,0,1,float16,fp8,0,0.05735466877619425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,64,128,1,float16,float16,0,0.717957337697347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,64,128,1,float16,fp8,0,0.7147466341654459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,64,0,1,float16,float16,0,1.0333493550618489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,64,128,1,fp8,fp8,0,0.9074239730834961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,64,0,1,float16,fp8,0,1.0224640369415283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,64,128,1,float16,float16,0,0.7345066865285238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,64,0,1,fp8,fp8,0,0.9418559869130453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,64,128,1,float16,fp8,0,0.7227839628855387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,64,0,1,float16,float16,0,1.0401066939036052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,64,128,1,fp8,fp8,0,0.9224373499552408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,64,0,1,float16,fp8,0,1.029194672902425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,64,0,1,fp8,fp8,0,0.9579306443532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,64,128,1,float16,float16,0,0.7621653079986572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,64,0,1,float16,float16,0,1.0707146326700847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,64,128,1,float16,fp8,0,0.7489066918691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,64,128,1,float16,float16,0,0.42364267508188885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,64,128,1,fp8,fp8,0,0.9491306940714518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,64,0,1,fp8,fp8,0,0.9835626284281412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,64,128,1,float16,fp8,0,0.4148266712824504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,64,0,1,float16,fp8,0,1.0542079607645671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,64,0,1,float16,float16,0,0.5818026860555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,64,128,1,fp8,fp8,0,0.5291680097579956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,64,0,1,float16,fp8,0,0.5727893511454264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,64,128,1,float16,float16,0,0.36347734928131104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,64,0,1,fp8,fp8,0,0.5393386681874593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,64,128,1,float16,fp8,0,0.3572266499201457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,64,0,1,float16,fp8,0,0.512223998705546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,64,0,1,float16,float16,0,0.5212639967600504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,64,128,1,fp8,fp8,0,0.4719146490097046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,64,128,1,float16,float16,0,0.37537598609924316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,64,0,1,fp8,fp8,0,0.48474133014678955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,64,0,1,float16,float16,0,0.5317013263702393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,64,128,1,float16,fp8,0,0.36982933680216473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,64,128,1,fp8,fp8,0,0.4812213182449341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,64,0,1,float16,fp8,0,0.5247946580251058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,64,0,1,fp8,fp8,0,0.4983359972635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,64,128,1,float16,float16,0,0.3840320110321045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,64,128,1,float16,fp8,0,0.38504000504811603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,64,0,1,float16,float16,0,0.541045347849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,64,128,1,fp8,fp8,0,0.4933706521987915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,64,128,1,float16,float16,0,0.2212373415629069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,64,0,1,float16,fp8,0,0.5364693403244019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,64,0,1,fp8,fp8,0,0.5049920082092285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,64,0,1,float16,float16,0,0.3036373257637024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,64,128,1,float16,fp8,0,0.21694932381312051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,64,128,1,fp8,fp8,0,0.2826933264732361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,64,0,1,float16,fp8,0,0.2969920039176941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,64,0,1,fp8,fp8,0,0.27712533871332806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,64,128,1,float16,float16,0,0.19082132975260416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,64,0,1,float16,float16,0,0.27153066794077557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,64,0,1,float16,float16,0,0.279477337996165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,64,128,1,float16,fp8,0,0.1890666683514913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,64,128,1,fp8,fp8,0,0.25809067487716675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,64,0,1,float16,fp8,0,0.269322673479716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,64,0,1,fp8,fp8,0,0.25190399090449017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,64,128,1,float16,float16,0,0.19529066483179727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,64,128,1,float16,fp8,0,0.1932106614112854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,64,128,1,fp8,fp8,0,0.25990933179855347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,64,0,1,float16,fp8,0,0.2770613431930542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,64,0,1,fp8,fp8,0,0.25380800167719525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,64,128,1,float16,float16,0,0.2015999952952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,64,0,1,float16,float16,0,0.2824479937553406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,64,128,1,float16,fp8,0,0.19882667064666748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,64,128,1,fp8,fp8,0,0.2661493420600891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,64,0,1,float16,fp8,0,0.279258668422699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,64,128,1,float16,float16,0,0.11949333548545837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,64,0,1,fp8,fp8,0,0.2621973355611165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,64,0,1,float16,float16,0,0.1544319987297058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,64,128,1,float16,fp8,0,0.11990933616956075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,64,128,1,fp8,fp8,0,0.1600053310394287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,64,0,1,float16,fp8,0,0.15285866459210715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,64,0,1,fp8,fp8,0,0.1472586691379547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,64,128,1,float16,float16,0,0.10666132966677348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,64,0,1,float16,float16,0,0.13970667123794556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,64,128,1,float16,fp8,0,0.10475732882817586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,64,128,1,fp8,fp8,0,0.14660267035166422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,64,0,1,float16,fp8,0,0.1378720005353292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,64,0,1,fp8,fp8,0,0.1334933340549469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,64,128,1,float16,float16,0,0.10690666238466899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,64,0,1,float16,float16,0,0.14005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,64,128,1,float16,fp8,0,0.10733333230018616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,64,128,1,fp8,fp8,0,0.1483519971370697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,64,0,1,float16,fp8,0,0.14017066359519958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,64,128,1,float16,fp8,0,0.1111840009689331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,64,0,1,fp8,fp8,0,0.13664000233014426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,64,128,1,float16,float16,0,0.11198932925860088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,64,0,1,float16,float16,0,0.14274666706720987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,64,0,1,float16,float16,0,0.08437333504358928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,64,128,1,fp8,fp8,0,0.15473066767056784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,64,0,1,float16,fp8,0,0.14288000265757242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,64,128,1,float16,float16,0,0.07064533233642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,64,0,1,fp8,fp8,0,0.141184002161026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,64,0,1,fp8,fp8,0,0.08603733777999878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,64,128,1,float16,fp8,0,0.07134933272997539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,64,128,1,fp8,fp8,0,0.09916266798973083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,64,0,1,float16,fp8,0,0.0842186709245046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,64,128,1,float16,float16,0,0.06359466910362244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,64,0,1,fp8,fp8,0,0.07416533430417378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,64,0,1,float16,float16,0,0.07826666533946991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,64,128,1,float16,fp8,0,0.06312533219655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,64,128,1,fp8,fp8,0,0.08095466593901317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,64,0,1,float16,fp8,0,0.07818133135636647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,64,0,1,float16,fp8,0,0.07781333227952321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,64,128,1,float16,float16,0,0.06401599943637848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,64,0,1,float16,float16,0,0.07918400069077809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,64,128,1,float16,fp8,0,0.06365333497524261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,64,128,1,fp8,fp8,0,0.08180800080299377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,64,0,1,fp8,fp8,0,0.0765119989713033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,64,128,1,float16,float16,0,0.06682666639486949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,64,0,1,float16,float16,0,0.08096533517042796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,64,128,1,float16,fp8,0,0.06624533236026764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,64,128,1,float16,fp8,0,0.04515733321507772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,64,128,1,fp8,fp8,0,0.09044800202051799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,64,0,1,float16,fp8,0,0.08039466540018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,64,0,1,fp8,fp8,0,0.04939733445644379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,64,0,1,fp8,fp8,0,0.07946666578451793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,64,128,1,float16,float16,0,0.044256001710891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,64,0,1,float16,float16,0,0.05089599887530009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,64,0,1,float16,fp8,0,0.04967466493447622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,64,128,1,fp8,fp8,0,0.05691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,64,0,1,float16,fp8,0,0.05115733544031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,64,128,1,float16,float16,0,0.04404266675313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,64,0,1,float16,float16,0,0.04956266780694326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,64,128,1,float16,fp8,0,0.043824002146720886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,64,128,1,float16,fp8,0,0.04410133262475332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,64,128,1,fp8,fp8,0,0.05428266525268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,64,0,1,fp8,fp8,0,0.047968000173568726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,64,128,1,float16,float16,0,0.04387733340263367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,64,0,1,float16,float16,0,0.04933333396911621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,64,128,1,float16,fp8,0,0.04509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,64,128,1,fp8,fp8,0,0.05522666871547699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,64,0,1,float16,fp8,0,0.04994666576385498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,64,0,1,fp8,fp8,0,0.047824000318845115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,64,128,1,float16,float16,0,0.04430399835109711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,64,0,1,float16,float16,0,0.04402133325735728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,64,0,1,float16,float16,0,0.04991999765237173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,64,128,1,fp8,fp8,0,0.05563200016816457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,64,0,1,float16,fp8,0,0.050581331054369606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,64,0,1,fp8,fp8,0,0.04905066887537638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,64,128,1,float16,float16,0,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,64,128,1,float16,fp8,0,0.03422933320204417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,64,128,1,fp8,fp8,0,0.040549332896868386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,64,128,1,fp8,fp8,0,0.04038933416207632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,64,0,1,float16,fp8,0,0.04414933423201243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,64,0,1,fp8,fp8,0,0.04281599819660187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,64,128,1,float16,float16,0,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,64,0,1,float16,float16,0,0.04385066529115041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,64,0,1,float16,float16,0,0.0439626673857371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,64,128,1,float16,fp8,0,0.033941333492596946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,64,0,1,float16,fp8,0,0.04400533437728882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,64,0,1,fp8,fp8,0,0.04287466903527578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,64,128,1,float16,float16,0,0.03401066611210505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,64,128,1,float16,fp8,0,0.03365333378314972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,64,128,1,fp8,fp8,0,0.04050666590531667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,64,128,1,fp8,fp8,0,0.04035733391841253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,64,0,1,float16,fp8,0,0.04393066465854645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,64,0,1,fp8,fp8,0,0.04321066538492838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,64,128,1,float16,float16,0,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,64,0,1,float16,float16,0,0.04374399781227112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,64,128,1,float16,fp8,0,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,64,0,1,float16,fp8,0,0.04404800136884054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,64,0,1,fp8,fp8,0,0.042863999803860985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,64,128,1,float16,float16,0,0.5400426785151163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,64,0,1,float16,float16,0,0.6783093611399332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,64,128,1,float16,fp8,0,0.5341973304748535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,64,128,1,fp8,fp8,0,0.6824373404184977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,64,0,1,float16,fp8,0,0.6718239784240723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,64,0,1,fp8,fp8,0,0.6197013457616171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,64,128,1,float16,float16,0,0.5532426834106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,64,0,1,float16,float16,0,0.6929653485616049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,64,128,1,float16,fp8,0,0.5431253512700399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,64,128,1,fp8,fp8,0,0.7000373204549154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,64,0,1,float16,fp8,0,0.6854720115661621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,64,0,1,fp8,fp8,0,0.6382559935251871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,64,128,1,float16,float16,0,0.571664015452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,64,128,1,float16,fp8,0,0.560474673906962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,64,0,1,float16,float16,0,0.7137119770050049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,64,128,1,float16,float16,0,0.32062933842341107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,64,128,1,fp8,fp8,0,0.7221013704935709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,64,0,1,float16,float16,0,0.3922079801559448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,64,0,1,float16,fp8,0,0.701258659362793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,64,0,1,fp8,fp8,0,0.6581759850184122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,64,128,1,float16,fp8,0,0.31512532631556195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,64,128,1,fp8,fp8,0,0.402672012646993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,64,0,1,float16,fp8,0,0.3837600151697795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,64,128,1,float16,float16,0,0.27743999163309735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,64,0,1,fp8,fp8,0,0.3665493329366048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,64,0,1,float16,float16,0,0.34992531935373944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,64,128,1,float16,fp8,0,0.2703733245531718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,64,128,1,fp8,fp8,0,0.35920532544453937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,64,0,1,float16,fp8,0,0.3432533343633016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,64,0,1,fp8,fp8,0,0.3169706662495931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,64,128,1,float16,float16,0,0.2843466599782308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,64,0,1,float16,float16,0,0.35583468278249103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,64,128,1,float16,fp8,0,0.2825919985771179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,64,128,1,fp8,fp8,0,0.36851199467976886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,64,0,1,float16,fp8,0,0.3522293170293172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,64,0,1,fp8,fp8,0,0.324346661567688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,64,128,1,float16,float16,0,0.2934719920158386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,64,0,1,float16,float16,0,0.3640373150507609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,64,128,1,float16,fp8,0,0.28995199998219806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,64,128,1,fp8,fp8,0,0.3821013371149699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,64,0,1,float16,fp8,0,0.36053868134816486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,64,128,1,float16,float16,0,0.17094399531682333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,64,0,1,fp8,fp8,0,0.337775985399882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,64,0,1,float16,float16,0,0.21850667397181192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,64,128,1,float16,fp8,0,0.1676479975382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,64,0,1,fp8,fp8,0,0.18813333908716837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,64,128,1,fp8,fp8,0,0.21962666511535645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,64,0,1,float16,fp8,0,0.2173759937286377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,64,128,1,float16,float16,0,0.14840533336003622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,64,0,1,float16,fp8,0,0.1732213298479716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,64,0,1,float16,float16,0,0.17473600308100382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,64,128,1,float16,fp8,0,0.14576533436775208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,64,128,1,fp8,fp8,0,0.1987733244895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,64,0,1,fp8,fp8,0,0.17011199394861856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,64,128,1,float16,float16,0,0.15345600247383118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,64,0,1,float16,float16,0,0.17786133289337158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,64,128,1,float16,float16,0,0.15659733613332114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,64,128,1,float16,fp8,0,0.15084266662597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,64,128,1,fp8,fp8,0,0.20433066288630167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,64,0,1,float16,fp8,0,0.17626132567723593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,64,0,1,fp8,fp8,0,0.17967466513315836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,64,0,1,fp8,fp8,0,0.17494400342305502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,64,0,1,float16,float16,0,0.184005339940389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,64,128,1,float16,fp8,0,0.15729066729545593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,64,128,1,fp8,fp8,0,0.20961600542068481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,64,128,1,fp8,fp8,0,0.12808000048001608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,64,0,1,float16,fp8,0,0.1836586594581604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,64,128,1,float16,float16,0,0.08362133304278056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,64,128,1,float16,float16,0,0.09474666913350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,64,0,1,float16,float16,0,0.10506666700045268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,64,128,1,float16,fp8,0,0.09545600414276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,64,0,1,float16,fp8,0,0.10682132840156555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,64,0,1,fp8,fp8,0,0.10603732864061992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,64,0,1,float16,float16,0,0.09474133451779683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,64,128,1,float16,fp8,0,0.08072533210118611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,64,128,1,fp8,fp8,0,0.11017599701881409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,64,0,1,float16,fp8,0,0.0934879978497823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,64,0,1,fp8,fp8,0,0.09423999985059102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,64,128,1,float16,float16,0,0.08553066849708557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,64,0,1,float16,float16,0,0.09672000010808308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,64,128,1,float16,fp8,0,0.08541867136955261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,64,128,1,fp8,fp8,0,0.11666133006413777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,64,0,1,float16,fp8,0,0.09523733456929524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,64,0,1,fp8,fp8,0,0.09477866689364116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,64,128,1,float16,float16,0,0.08749866485595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,64,0,1,fp8,fp8,0,0.09908800323804219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,64,0,1,float16,float16,0,0.10009066263834636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,64,128,1,float16,fp8,0,0.08825066685676575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,64,128,1,fp8,fp8,0,0.12272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,64,0,1,float16,fp8,0,0.10030399759610494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,64,128,1,float16,float16,0,0.05425600210825602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,64,0,1,float16,float16,0,0.06061333417892456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,64,128,1,float16,fp8,0,0.055904000997543335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,64,128,1,fp8,fp8,0,0.08000533282756805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,64,0,1,float16,fp8,0,0.062447999914487205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,64,0,1,fp8,fp8,0,0.06158400078614553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,64,128,1,float16,float16,0,0.05089066425959269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,64,0,1,fp8,fp8,0,0.0554666668176651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,64,0,1,float16,float16,0,0.05774933099746704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,64,128,1,float16,fp8,0,0.050442665815353394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,64,128,1,fp8,fp8,0,0.06869866450627644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,64,128,1,fp8,fp8,0,0.06845333178838094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,64,0,1,float16,fp8,0,0.05644266804059347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,64,128,1,float16,float16,0,0.05175999800364176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,64,0,1,float16,float16,0,0.05729066828886668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,64,128,1,float16,fp8,0,0.05130666494369507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,64,0,1,float16,fp8,0,0.05698133508364359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,64,0,1,fp8,fp8,0,0.05608533322811127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,64,128,1,float16,float16,0,0.052517334620157875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,64,0,1,float16,float16,0,0.057578667998313904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,64,128,1,float16,fp8,0,0.05263466636339823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,64,128,1,fp8,fp8,0,0.06702400247255962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,64,0,1,float16,fp8,0,0.058506667613983154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,64,0,1,fp8,fp8,0,0.05681066711743673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,64,128,1,float16,float16,0,0.0408746674656868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,64,0,1,float16,float16,0,0.03995199998219808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,64,128,1,float16,fp8,0,0.041093334555625916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,64,128,1,fp8,fp8,0,0.04529599845409393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,64,0,1,float16,fp8,0,0.040037333965301514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,64,0,1,fp8,fp8,0,0.038165333370367684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,64,128,1,float16,float16,0,0.039520000418027244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,64,0,1,float16,float16,0,0.03878399978081385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,64,128,1,float16,fp8,0,0.03972266614437103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,64,128,1,fp8,fp8,0,0.0443146675825119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,64,0,1,float16,fp8,0,0.03908266623814901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,64,0,1,fp8,fp8,0,0.03681600093841553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,64,128,1,float16,float16,0,0.04053866614898046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,64,0,1,float16,float16,0,0.03865066667397817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,64,128,1,float16,fp8,0,0.040378667414188385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,64,128,1,fp8,fp8,0,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,64,128,1,float16,fp8,0,0.037248000502586365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,64,0,1,float16,fp8,0,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,64,0,1,fp8,fp8,0,0.03697066754102707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,64,128,1,float16,float16,0,0.03751999884843826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,64,0,1,float16,float16,0,0.03972266614437103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,64,128,1,fp8,fp8,0,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,64,0,1,float16,fp8,0,0.04020266731580099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,64,0,1,fp8,fp8,0,0.037231999138991036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,64,128,1,float16,float16,0,0.030970667799313862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,64,0,1,float16,float16,0,0.03706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,64,128,1,float16,fp8,0,0.03143466760714849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,64,128,1,fp8,fp8,0,0.03772266705830892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,64,0,1,float16,fp8,0,0.03777066618204117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,64,0,1,fp8,fp8,0,0.036464000741640724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,64,128,1,float16,float16,0,0.030810666580994923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,64,0,1,float16,float16,0,0.036570665736993156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,64,128,1,float16,fp8,0,0.03054400036732356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,64,128,1,fp8,fp8,0,0.037461332976818085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,64,0,1,float16,fp8,0,0.037077332536379494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,64,0,1,float16,fp8,0,0.03665599972009659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,64,0,1,fp8,fp8,0,0.036117332677046456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,64,128,1,float16,float16,0,0.031184000273545582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,64,0,1,float16,float16,0,0.03702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,64,128,1,float16,fp8,0,0.031082667410373688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,64,128,1,float16,fp8,0,0.030896000564098358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,64,128,1,fp8,fp8,0,0.03749333322048187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,64,0,1,fp8,fp8,0,0.03628266602754593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,64,128,1,float16,float16,0,0.030965333183606465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,64,0,1,float16,float16,0,0.03695466617743174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,64,128,1,fp8,fp8,0,0.03762666632731756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,64,0,1,float16,fp8,0,0.0367999995748202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,64,0,1,fp8,fp8,0,0.03628266602754593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,64,128,1,float16,float16,0,0.6169759829839071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,64,0,1,float16,float16,0,0.6939840316772461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,64,128,1,float16,fp8,0,0.6158453226089478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,64,128,1,fp8,fp8,0,0.8251626491546631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,64,0,1,float16,fp8,0,0.6948800086975098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,64,0,1,fp8,fp8,0,0.6502613226572672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,64,128,1,float16,float16,0,0.619327982266744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,64,0,1,float16,float16,0,0.6977120240529379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,64,128,1,float16,fp8,0,0.6204320192337036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,64,0,1,float16,fp8,0,0.6974720160166422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,64,128,1,fp8,fp8,0,0.8306293487548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,64,0,1,fp8,fp8,0,0.6549546718597412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,64,128,1,float16,float16,0,0.6516746679941813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,64,0,1,float16,float16,0,0.732256015141805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,64,128,1,float16,fp8,0,0.6404480139414469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,64,128,1,fp8,fp8,0,0.8631733258565267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,64,128,1,float16,float16,0,0.37624533971150714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,64,128,1,float16,fp8,0,0.36956799030303955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,64,0,1,float16,fp8,0,0.719055970509847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,64,0,1,float16,float16,0,0.4156159957249959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,64,0,1,fp8,fp8,0,0.6898773511250814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,64,128,1,fp8,fp8,0,0.46048001448313397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,64,128,1,float16,float16,0,0.31995733578999835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,64,0,1,float16,fp8,0,0.4084320068359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,64,0,1,fp8,fp8,0,0.3689546585083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,64,0,1,float16,float16,0,0.36292799313863117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,64,128,1,float16,fp8,0,0.3208106756210327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,64,128,1,fp8,fp8,0,0.4291679859161377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,64,0,1,float16,fp8,0,0.3588000138600667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,64,0,1,fp8,fp8,0,0.3341546853383382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,64,128,1,float16,float16,0,0.3219199975331624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,64,0,1,float16,float16,0,0.36309866110483807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,64,128,1,float16,fp8,0,0.3208586573600769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,64,128,1,fp8,fp8,0,0.43541868527730304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,64,0,1,float16,fp8,0,0.3631093502044678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,64,0,1,fp8,fp8,0,0.3421706755956014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,64,128,1,float16,float16,0,0.3349440097808838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,64,128,1,fp8,fp8,0,0.4386719862620036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,64,0,1,float16,float16,0,0.37570667266845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,64,128,1,float16,fp8,0,0.3317173322041829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,64,0,1,float16,fp8,0,0.3720426559448242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,64,0,1,fp8,fp8,0,0.34562134742736816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,64,128,1,float16,float16,0,0.20350933074951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,64,0,1,float16,float16,0,0.22040534019470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,64,128,1,float16,fp8,0,0.19732266664505005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,64,128,1,float16,float16,0,0.17470399538675943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,64,128,1,fp8,fp8,0,0.24962133169174194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,64,0,1,float16,fp8,0,0.21760533253351846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,64,0,1,fp8,fp8,0,0.1920213301976522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,64,0,1,float16,float16,0,0.19780800739924112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,64,128,1,float16,fp8,0,0.17427200078964233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,64,128,1,fp8,fp8,0,0.23425066471099854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,64,0,1,float16,fp8,0,0.19518399238586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,64,128,1,float16,fp8,0,0.17603733142217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,64,0,1,fp8,fp8,0,0.1764906644821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,64,128,1,float16,float16,0,0.1762346625328064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,64,0,1,float16,float16,0,0.19694934288660684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,64,128,1,fp8,fp8,0,0.23644266525904337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,64,0,1,float16,fp8,0,0.1953493356704712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,64,0,1,fp8,fp8,0,0.17869865894317627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,64,128,1,float16,float16,0,0.1827039917310079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,64,0,1,float16,float16,0,0.20202666521072388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,64,128,1,float16,fp8,0,0.18275733788808188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,64,128,1,fp8,fp8,0,0.24016533295313516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,64,0,1,float16,fp8,0,0.20175466934839884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,64,0,1,fp8,fp8,0,0.18153067429860434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,64,128,1,float16,float16,0,0.11486933628718059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,64,0,1,float16,float16,0,0.11921599507331848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,64,128,1,float16,fp8,0,0.1127946674823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,64,128,1,fp8,fp8,0,0.14524267117182413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,64,0,1,float16,fp8,0,0.11635200182596843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,64,0,1,fp8,fp8,0,0.1074720025062561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,64,128,1,float16,float16,0,0.0983786682287852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,64,0,1,float16,float16,0,0.10353066523869832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,64,128,1,float16,fp8,0,0.09852266311645508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,64,128,1,fp8,fp8,0,0.13433600465456644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,64,0,1,float16,fp8,0,0.10357333223025005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,64,0,1,fp8,fp8,0,0.0981066624323527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,64,128,1,float16,float16,0,0.09965333342552185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,64,0,1,float16,float16,0,0.1055573324362437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,64,128,1,float16,fp8,0,0.10090667009353638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,64,128,1,fp8,fp8,0,0.13658666610717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,64,0,1,float16,fp8,0,0.10617066423098247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,64,0,1,fp8,fp8,0,0.09875200192133586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,64,128,1,float16,float16,0,0.10335999727249146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,64,0,1,float16,float16,0,0.10930132865905762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,64,0,1,float16,float16,0,0.06643733382225037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,64,128,1,float16,fp8,0,0.10468799869219463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,64,128,1,fp8,fp8,0,0.13925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,64,0,1,float16,fp8,0,0.10853866736094157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,64,0,1,fp8,fp8,0,0.10132267077763875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,64,128,1,float16,float16,0,0.06703466673692067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,64,128,1,float16,fp8,0,0.06429333488146464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,64,128,1,fp8,fp8,0,0.09070400396982829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,64,0,1,float16,fp8,0,0.06540800134340923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,64,0,1,fp8,fp8,0,0.06437866886456807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,64,128,1,float16,float16,0,0.05727999905745188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,64,0,1,float16,float16,0,0.05934933324654897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,64,128,1,float16,fp8,0,0.057850668827692665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,64,128,1,fp8,fp8,0,0.0774293343226115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,64,0,1,float16,fp8,0,0.05945600072542826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,64,0,1,fp8,fp8,0,0.05580266813437144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,64,128,1,float16,float16,0,0.05810666580994924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,64,0,1,float16,float16,0,0.05975999931494395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,64,128,1,float16,fp8,0,0.05839466551939646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,64,128,1,fp8,fp8,0,0.07861333092053731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,64,0,1,float16,fp8,0,0.05931200087070465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,64,0,1,fp8,fp8,0,0.05668266614278158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,64,128,1,float16,float16,0,0.06101333101590475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,64,0,1,float16,float16,0,0.063509335120519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,64,128,1,float16,fp8,0,0.0603359987338384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,64,128,1,fp8,fp8,0,0.08193600177764893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,64,0,1,float16,fp8,0,0.061247999469439186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,64,0,1,fp8,fp8,0,0.05961066484451294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,64,128,1,float16,float16,0,0.04170133173465729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,64,0,1,float16,float16,0,0.04078399886687597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,64,128,1,float16,fp8,0,0.04222933451334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,64,128,1,fp8,fp8,0,0.051674668987592064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,64,0,1,float16,fp8,0,0.04008533308903376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,64,0,1,fp8,fp8,0,0.03659733384847641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,64,128,1,float16,float16,0,0.03862400104602178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,64,0,1,float16,float16,0,0.03765333443880081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,64,0,1,float16,float16,0,0.03793599953254064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,64,128,1,float16,fp8,0,0.03988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,64,128,1,fp8,fp8,0,0.05064000189304352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,64,0,1,float16,fp8,0,0.037263999382654824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,64,0,1,fp8,fp8,0,0.035487999518712364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,64,0,1,fp8,fp8,0,0.03510399907827377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,64,128,1,float16,float16,0,0.040448000033696495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,64,128,1,float16,fp8,0,0.03967999915281931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,64,128,1,fp8,fp8,0,0.05115200082461039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,64,0,1,float16,fp8,0,0.03789333254098892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,64,128,1,float16,float16,0,0.041706666350364685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,64,0,1,float16,float16,0,0.038592000802357994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,64,128,1,float16,fp8,0,0.041050667564074196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,64,128,1,fp8,fp8,0,0.05128000179926554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,64,0,1,float16,fp8,0,0.039520000418027244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,64,0,1,fp8,fp8,0,0.035962666074434914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,64,128,1,float16,float16,0,0.02771199991305669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,64,0,1,float16,float16,0,0.030058667063713074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,64,128,1,float16,fp8,0,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,64,128,1,fp8,fp8,0,0.03445333242416382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,64,0,1,float16,fp8,0,0.02959466725587845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,64,0,1,fp8,fp8,0,0.028789333999156952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,64,128,1,float16,float16,0,0.026464000344276428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,64,0,1,float16,float16,0,0.028762665887673695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,64,128,1,float16,fp8,0,0.026602665583292644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,64,128,1,fp8,fp8,0,0.03254399945338567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,64,0,1,float16,fp8,0,0.028832000990708668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,64,0,1,fp8,fp8,0,0.026928000152111053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,64,128,1,float16,float16,0,0.026922665536403656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,64,0,1,float16,float16,0,0.028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,64,128,1,float16,fp8,0,0.02703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,64,128,1,fp8,fp8,0,0.033146666983763375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,64,128,1,float16,fp8,0,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,64,0,1,float16,fp8,0,0.02849599967400233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,64,0,1,fp8,fp8,0,0.027776000400384266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,64,128,1,float16,float16,0,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,64,128,1,float16,float16,0,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,64,0,1,float16,float16,0,0.029706666866938274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,64,128,1,fp8,fp8,0,0.03425599883000056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,64,0,1,float16,fp8,0,0.02924799919128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,64,0,1,fp8,fp8,0,0.028384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,64,0,1,float16,float16,0,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,64,128,1,float16,fp8,0,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,64,128,1,fp8,fp8,0,0.031343999008337654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,64,0,1,float16,fp8,0,0.027274665733178455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,64,0,1,fp8,fp8,0,0.026672000686327618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,64,128,1,float16,float16,0,0.02380799998839696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,64,0,1,float16,float16,0,0.026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,64,128,1,float16,fp8,0,0.024362665911515553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,64,128,1,fp8,fp8,0,0.031136001149813335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,64,0,1,float16,fp8,0,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,64,0,1,fp8,fp8,0,0.025909334421157837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,64,128,1,float16,float16,0,0.024154665569464367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,64,0,1,float16,float16,0,0.02672533442576726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,64,128,1,float16,fp8,0,0.024517332514127094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,64,128,1,fp8,fp8,0,0.031194667021433514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,64,0,1,float16,fp8,0,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,64,0,1,fp8,fp8,0,0.026170666019121807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,64,128,1,float16,float16,0,0.024469333390394848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,64,0,1,float16,float16,0,0.026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,64,128,1,float16,fp8,0,0.02470933397610982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,64,128,1,fp8,fp8,0,0.03108799954255422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,64,0,1,float16,fp8,0,0.027322667340437572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,64,0,1,fp8,fp8,0,0.026202666262785595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,64,128,1,float16,float16,0,0.5860533316930135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,64,0,1,float16,float16,0,0.5714879830678304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,64,128,1,float16,fp8,0,0.5834293365478516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,64,0,1,float16,fp8,0,0.5727466742197672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,64,0,1,fp8,fp8,0,0.5235999822616577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,64,128,1,fp8,fp8,0,0.7813066641489664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,64,128,1,float16,float16,0,0.5914080142974854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,64,0,1,float16,float16,0,0.5769280195236206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,64,128,1,float16,fp8,0,0.5881066719690958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,64,0,1,float16,fp8,0,0.5775359869003296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,64,0,1,fp8,fp8,0,0.5348586638768514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,64,128,1,fp8,fp8,0,0.7916639645894369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,64,128,1,float16,float16,0,0.618229349454244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,64,0,1,float16,float16,0,0.6043946743011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,64,128,1,float16,fp8,0,0.6099840005238851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,64,128,1,float16,float16,0,0.36260799566904706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,64,0,1,float16,fp8,0,0.5943573315938314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,64,128,1,fp8,fp8,0,0.8063999811808268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,64,0,1,fp8,fp8,0,0.5567040046056112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,64,0,1,float16,float16,0,0.3530506690343221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,64,128,1,fp8,fp8,0,0.4410613377888997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,64,128,1,float16,fp8,0,0.35328535238901776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,64,0,1,float16,fp8,0,0.34351468086242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,64,0,1,fp8,fp8,0,0.3012640078862508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,64,128,1,float16,float16,0,0.3056853413581848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,64,0,1,float16,float16,0,0.2969866593678792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,64,128,1,float16,fp8,0,0.30668799082438153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,64,128,1,fp8,fp8,0,0.4038826624552409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,64,0,1,float16,fp8,0,0.2955626646677653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,64,0,1,fp8,fp8,0,0.2717120051383972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,64,128,1,float16,float16,0,0.3076853354771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,64,0,1,float16,float16,0,0.2993546724319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,64,128,1,float16,fp8,0,0.3076533277829488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,64,128,1,fp8,fp8,0,0.41127999623616535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,64,0,1,float16,fp8,0,0.2997973362604777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,64,0,1,fp8,fp8,0,0.27746667464574176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,64,128,1,float16,float16,0,0.3198240001996358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,64,0,1,float16,float16,0,0.3120693365732829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,64,128,1,float16,fp8,0,0.3176586627960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,64,128,1,fp8,fp8,0,0.41630399227142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,64,0,1,float16,fp8,0,0.30882134040196735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,64,128,1,float16,float16,0,0.19448532660802206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,64,0,1,fp8,fp8,0,0.16087466478347778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,64,0,1,fp8,fp8,0,0.28759467601776123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,64,0,1,float16,float16,0,0.18742932875951132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,64,0,1,float16,float16,0,0.16126400232315063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,64,128,1,float16,fp8,0,0.1901386578877767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,64,128,1,fp8,fp8,0,0.23753599325815836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,64,0,1,float16,fp8,0,0.18344533443450928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,64,128,1,float16,float16,0,0.16777600844701132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,64,128,1,float16,fp8,0,0.1665440003077189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,64,0,1,float16,fp8,0,0.16123732924461365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,64,128,1,fp8,fp8,0,0.22130666176478067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,64,0,1,fp8,fp8,0,0.14706133802731833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,64,128,1,float16,float16,0,0.16833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,64,0,1,float16,float16,0,0.1630400021870931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,64,128,1,float16,fp8,0,0.168778657913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,64,128,1,fp8,fp8,0,0.2246239980061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,64,0,1,float16,fp8,0,0.16312533617019653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,64,0,1,fp8,fp8,0,0.1492586632569631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,64,128,1,float16,float16,0,0.17484800020853677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,64,0,1,float16,float16,0,0.1678933302561442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,64,128,1,float16,fp8,0,0.17413866519927979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,64,128,1,fp8,fp8,0,0.22722667455673218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,64,0,1,float16,fp8,0,0.16750933726628622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,64,0,1,fp8,fp8,0,0.15260799725850424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,64,128,1,float16,float16,0,0.10899733503659566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,64,0,1,float16,float16,0,0.10765332976977031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,64,128,1,float16,fp8,0,0.10771733522415161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,64,128,1,fp8,fp8,0,0.13751467068990073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,64,0,1,float16,fp8,0,0.1053653359413147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,64,0,1,fp8,fp8,0,0.09309333562850952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,64,128,1,float16,float16,0,0.0942186713218689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,64,0,1,float16,float16,0,0.08897599577903748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,64,128,1,float16,fp8,0,0.09438400467236836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,64,128,1,float16,float16,0,0.09691733121871948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,64,128,1,fp8,fp8,0,0.1271573305130005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,64,0,1,float16,fp8,0,0.08906666437784831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,64,0,1,fp8,fp8,0,0.08220799763997395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,64,0,1,float16,float16,0,0.09044266740481059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,64,128,1,float16,fp8,0,0.09595732887585957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,64,128,1,fp8,fp8,0,0.12981866796811423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,64,128,1,float16,fp8,0,0.1002400020758311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,64,0,1,float16,fp8,0,0.09008000294367473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,64,0,1,float16,fp8,0,0.09275199969609578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,64,0,1,fp8,fp8,0,0.08427733182907104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,64,128,1,float16,float16,0,0.10103467106819153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,64,0,1,float16,float16,0,0.09318400422732036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,64,128,1,fp8,fp8,0,0.1325759987036387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,64,0,1,fp8,fp8,0,0.08692266543706258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,64,128,1,float16,float16,0,0.06396799782911937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,64,0,1,float16,float16,0,0.05832533538341522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,64,128,1,float16,fp8,0,0.06251733501752217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,64,128,1,fp8,fp8,0,0.08647466699282329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,64,0,1,float16,fp8,0,0.057573333382606506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,64,0,1,fp8,fp8,0,0.0552106648683548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,64,128,1,float16,float16,0,0.05598400036493937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,64,0,1,float16,float16,0,0.05171200136343638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,64,128,1,float16,fp8,0,0.054885332783063255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,64,128,1,fp8,fp8,0,0.07212799787521362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,64,0,1,float16,fp8,0,0.05134933193524679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,64,0,1,fp8,fp8,0,0.04762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,64,128,1,float16,float16,0,0.056458666920661926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,64,128,1,float16,fp8,0,0.05690133571624756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,64,0,1,float16,float16,0,0.05284800132115682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,64,128,1,fp8,fp8,0,0.07367999851703644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,64,0,1,float16,fp8,0,0.05218133330345154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,64,0,1,fp8,fp8,0,0.04778666794300079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,64,128,1,float16,float16,0,0.057664001981417336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,64,0,1,float16,float16,0,0.0551146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,64,128,1,float16,fp8,0,0.05949333310127258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,64,128,1,fp8,fp8,0,0.0814879983663559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,64,0,1,float16,fp8,0,0.054789334535598755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,64,0,1,fp8,fp8,0,0.05134933193524679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,64,128,1,float16,float16,0,0.041322665909926094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,64,0,1,float16,float16,0,0.035605333745479584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,64,128,1,float16,fp8,0,0.041738669077555336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,64,128,1,fp8,fp8,0,0.05153599878152212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,64,0,1,float16,fp8,0,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,64,0,1,fp8,fp8,0,0.032186667124430336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,64,128,1,float16,float16,0,0.03858133405447006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,64,0,1,float16,float16,0,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,64,128,1,float16,fp8,0,0.0386559988061587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,64,128,1,fp8,fp8,0,0.049098665515581764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,64,0,1,float16,fp8,0,0.032858667274316154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,64,0,1,fp8,fp8,0,0.031221332649389904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,64,128,1,float16,float16,0,0.03828266759713491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,64,0,1,float16,float16,0,0.03384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,64,128,1,float16,fp8,0,0.03920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,64,128,1,fp8,fp8,0,0.05003733436266581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,64,0,1,float16,fp8,0,0.03356266766786575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,64,0,1,fp8,fp8,0,0.030805334448814392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,64,128,1,float16,float16,0,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,64,0,1,float16,float16,0,0.03523733218510946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,64,128,1,float16,fp8,0,0.04110399881998698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,64,128,1,fp8,fp8,0,0.051072001457214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,64,0,1,float16,fp8,0,0.03465066601832708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,64,0,1,fp8,fp8,0,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,64,128,1,float16,float16,0,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,64,0,1,float16,float16,0,0.024853333830833435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,64,128,1,float16,fp8,0,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,64,128,1,fp8,fp8,0,0.034389334420363106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,64,0,1,float16,fp8,0,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,64,0,1,fp8,fp8,0,0.02402133246262868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,64,128,1,float16,float16,0,0.026005332668622334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,64,0,1,float16,float16,0,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,64,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,64,128,1,float16,fp8,0,0.026698666314284008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,64,128,1,fp8,fp8,0,0.03265066693226496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,64,0,1,float16,fp8,0,0.024362665911515553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,64,0,1,fp8,fp8,0,0.022991999983787537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,64,128,1,float16,float16,0,0.026261332134405773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,64,128,1,float16,fp8,0,0.02685333291689555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,64,128,1,fp8,fp8,0,0.03422933320204417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,64,0,1,float16,fp8,0,0.024874667326609295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,64,0,1,fp8,fp8,0,0.022965334355831146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,64,128,1,float16,float16,0,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,64,0,1,float16,float16,0,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,64,128,1,float16,fp8,0,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,64,128,1,fp8,fp8,0,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,64,0,1,float16,fp8,0,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,64,0,1,fp8,fp8,0,0.024192000428835552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,64,128,1,float16,float16,0,0.024373332659403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,64,0,1,float16,float16,0,0.022602667411168415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,64,128,1,float16,fp8,0,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,64,128,1,fp8,fp8,0,0.031541332602500916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,64,0,1,float16,fp8,0,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,64,0,1,fp8,fp8,0,0.02239466706911723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,64,128,1,float16,float16,0,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,64,0,1,float16,float16,0,0.02235200007756551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,64,128,1,float16,fp8,0,0.024442667762438457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,64,128,1,fp8,fp8,0,0.031045332551002502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,64,0,1,float16,fp8,0,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,64,0,1,fp8,fp8,0,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,64,128,1,float16,float16,0,0.024298667907714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,64,0,1,float16,float16,0,0.023168000082174938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,64,128,1,float16,fp8,0,0.024400000770886738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,64,128,1,fp8,fp8,0,0.030853333572546642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,64,128,1,float16,fp8,0,0.024282666544119518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,64,0,1,float16,fp8,0,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,64,0,1,fp8,fp8,0,0.02214933435122172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,64,128,1,float16,float16,0,0.024085332949956257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,64,0,1,float16,float16,0,0.022687998910744984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,64,128,1,fp8,fp8,0,0.030832000076770782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,64,0,1,float16,fp8,0,0.02248000105222066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,64,0,1,fp8,fp8,0,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,64,128,1,float16,float16,0,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,64,0,1,float16,float16,0,0.021749332547187805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,64,128,1,float16,fp8,0,0.02385599911212921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,64,128,1,fp8,fp8,0,0.0301706666747729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,64,0,1,float16,fp8,0,0.02163733293612798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,64,0,1,float16,fp8,0,0.02182399978240331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,64,0,1,fp8,fp8,0,0.02067199970285098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,64,128,1,float16,float16,0,0.022837333381175995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,64,0,1,float16,float16,0,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,64,128,1,float16,fp8,0,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,64,128,1,fp8,fp8,0,0.030181333422660828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,64,0,1,fp8,fp8,0,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,64,128,1,float16,float16,0,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,64,0,1,float16,float16,0,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,64,128,1,float16,fp8,0,0.023541333774725597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,64,128,1,float16,fp8,0,0.024005333582560223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,64,128,1,fp8,fp8,0,0.02977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,64,0,1,float16,fp8,0,0.022085333863894146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,64,0,1,fp8,fp8,0,0.020842666427294414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,64,128,1,float16,float16,0,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,64,0,1,float16,float16,0,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,64,128,1,fp8,fp8,0,0.030031998952229817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,64,0,1,float16,fp8,0,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,64,0,1,fp8,fp8,0,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,64,128,1,float16,float16,0,0.2627360026041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,64,0,1,float16,float16,0,0.257695992787679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,64,128,1,float16,fp8,0,0.26259734233220416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,64,128,1,fp8,fp8,0,0.34705066680908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,64,0,1,float16,fp8,0,0.2568746606508891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,64,0,1,fp8,fp8,0,0.24017600218454996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,64,128,1,float16,float16,0,0.2688639958699544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,64,0,1,float16,float16,0,0.2638559937477112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,64,128,1,float16,fp8,0,0.2679786682128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,64,128,1,fp8,fp8,0,0.3513760169347127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,64,0,1,float16,fp8,0,0.26311467091242474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,64,0,1,fp8,fp8,0,0.2399946649869283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,64,128,1,float16,float16,0,0.2829759915669759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,64,0,1,float16,float16,0,0.2802826762199402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,64,128,1,float16,fp8,0,0.2802559932072957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,64,128,1,fp8,fp8,0,0.35596799850463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,64,128,1,float16,float16,0,0.1784693400065104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,64,0,1,float16,fp8,0,0.2741493384043376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,64,0,1,float16,float16,0,0.1765706737836202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,64,0,1,float16,fp8,0,0.17174933354059854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,64,0,1,fp8,fp8,0,0.24684266249338785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,64,128,1,float16,fp8,0,0.1736639936765035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,64,128,1,fp8,fp8,0,0.19878933827082315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,64,0,1,fp8,fp8,0,0.14285866419474283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,64,128,1,float16,float16,0,0.14332800110181174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,64,0,1,float16,float16,0,0.1399679978688558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,64,128,1,float16,fp8,0,0.14296533664067587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,64,128,1,fp8,fp8,0,0.1844266653060913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,64,0,1,float16,fp8,0,0.13983466227849325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,64,0,1,fp8,fp8,0,0.13192533453305563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,64,128,1,float16,float16,0,0.14781333009401956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,64,0,1,float16,float16,0,0.1453120013078054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,64,128,1,float16,fp8,0,0.14721600214640299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,64,0,1,float16,float16,0,0.15172800421714783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,64,128,1,fp8,fp8,0,0.18681599696477255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,64,0,1,float16,fp8,0,0.14541332920392355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,64,0,1,fp8,fp8,0,0.13403733571370444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,64,128,1,float16,float16,0,0.1557866632938385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,64,0,1,fp8,fp8,0,0.1351573367913564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,64,128,1,float16,float16,0,0.10240000486373901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,64,128,1,float16,fp8,0,0.15380799770355225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,64,128,1,fp8,fp8,0,0.19033600886662802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,64,0,1,float16,fp8,0,0.15126400192578635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,64,0,1,fp8,fp8,0,0.08323200047016144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,64,0,1,float16,float16,0,0.10062399506568909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,64,128,1,float16,fp8,0,0.09923199812571208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,64,128,1,fp8,fp8,0,0.11010666688283284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,64,0,1,float16,fp8,0,0.09880000352859497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,64,128,1,float16,float16,0,0.07611733178297679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,64,0,1,fp8,fp8,0,0.07356266677379608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,64,0,1,float16,fp8,0,0.07435733576615651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,64,0,1,float16,float16,0,0.07422400017579396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,64,128,1,float16,fp8,0,0.07524799803892772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,64,128,1,fp8,fp8,0,0.10058666268984477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,64,128,1,float16,float16,0,0.07832533121109009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,64,0,1,float16,float16,0,0.07689066727956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,64,128,1,float16,fp8,0,0.07772266864776611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,64,128,1,fp8,fp8,0,0.10260799527168274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,64,0,1,float16,fp8,0,0.07649600009123485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,64,128,1,float16,fp8,0,0.0841919978459676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,64,0,1,fp8,fp8,0,0.07457066575686137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,64,128,1,float16,float16,0,0.08571733037630717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,64,0,1,float16,float16,0,0.08412800232569377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,64,128,1,fp8,fp8,0,0.10552000006039937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,64,0,1,float16,fp8,0,0.0828906645377477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,64,0,1,fp8,fp8,0,0.07836266855398814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,64,128,1,float16,float16,0,0.050399998823801674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,64,0,1,float16,float16,0,0.04907733201980591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,64,128,1,float16,fp8,0,0.048485333720842995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,64,128,1,fp8,fp8,0,0.06558933357397716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,64,0,1,float16,fp8,0,0.048197334011395775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,64,0,1,fp8,fp8,0,0.050053333242734276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,64,128,1,float16,float16,0,0.0424586683511734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,64,0,1,float16,float16,0,0.042303999265034996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,64,128,1,float16,fp8,0,0.042453333735466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,64,128,1,fp8,fp8,0,0.05666666726271311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,64,0,1,float16,fp8,0,0.04199466605981191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,64,0,1,fp8,fp8,0,0.04298666616280874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,64,128,1,float16,float16,0,0.04357333481311798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,64,0,1,float16,float16,0,0.043007999658584595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,64,128,1,float16,fp8,0,0.04330133398373922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,64,128,1,fp8,fp8,0,0.056159997979799904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,64,0,1,float16,fp8,0,0.0422026664018631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,64,128,1,fp8,fp8,0,0.060965334375699363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,64,0,1,fp8,fp8,0,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,64,128,1,float16,float16,0,0.04504533112049103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,64,0,1,float16,float16,0,0.04463466505209605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,64,128,1,float16,fp8,0,0.04507199923197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,64,128,1,float16,fp8,0,0.034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,64,0,1,float16,fp8,0,0.04445866743723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,64,0,1,fp8,fp8,0,0.03068266560633977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,64,0,1,fp8,fp8,0,0.046570668617884316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,64,128,1,float16,float16,0,0.03475733349720637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,64,0,1,float16,float16,0,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,64,128,1,fp8,fp8,0,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,64,0,1,float16,fp8,0,0.03389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,64,128,1,float16,float16,0,0.03148799886306127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,64,0,1,float16,float16,0,0.03071466585000356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,64,128,1,float16,fp8,0,0.03149333347876867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,64,128,1,fp8,fp8,0,0.0349440003434817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,64,0,1,float16,fp8,0,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,64,0,1,float16,fp8,0,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,64,0,1,fp8,fp8,0,0.02884799987077713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,64,128,1,float16,float16,0,0.03229333211978277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,64,0,1,float16,float16,0,0.03305066625277201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,64,128,1,float16,fp8,0,0.032058666149775185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,64,128,1,fp8,fp8,0,0.035530666510264076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,64,0,1,fp8,fp8,0,0.028736000259717304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,64,128,1,float16,float16,0,0.03299733251333237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,64,0,1,float16,float16,0,0.03297599901755651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,64,128,1,float16,fp8,0,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,64,128,1,fp8,fp8,0,0.03618133316437403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,64,0,1,float16,fp8,0,0.032816000282764435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,64,0,1,fp8,fp8,0,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,64,128,1,float16,float16,0,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,64,0,1,float16,float16,0,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,64,128,1,float16,fp8,0,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,64,128,1,fp8,fp8,0,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,64,0,1,float16,fp8,0,0.021551998953024547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,64,0,1,fp8,fp8,0,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,64,128,1,float16,float16,0,0.020373333245515823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,64,0,1,float16,float16,0,0.019909333437681198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,64,128,1,float16,fp8,0,0.019962667177120846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,64,128,1,fp8,fp8,0,0.024085332949956257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,64,0,1,float16,fp8,0,0.020154666155576706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,64,0,1,fp8,fp8,0,0.0204373337328434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,64,128,1,float16,float16,0,0.020106667031844456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,64,0,1,float16,float16,0,0.020021333048741024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,64,128,1,float16,fp8,0,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,64,128,1,fp8,fp8,0,0.0245919997493426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,64,0,1,float16,fp8,0,0.020410666863123577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,64,0,1,fp8,fp8,0,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,64,128,1,float16,float16,0,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,64,0,1,float16,float16,0,0.020703999946514767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,64,128,1,float16,fp8,0,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,64,128,1,fp8,fp8,0,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,64,0,1,float16,fp8,0,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,64,128,1,fp8,fp8,0,0.022869333624839783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,64,0,1,float16,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,64,0,1,fp8,fp8,0,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,64,0,1,fp8,fp8,0,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,64,128,1,float16,float16,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,64,0,1,float16,float16,0,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,64,128,1,float16,fp8,0,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,64,0,1,float16,float16,0,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,64,128,1,float16,float16,0,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,64,128,1,float16,fp8,0,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,64,128,1,fp8,fp8,0,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,64,0,1,float16,fp8,0,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,64,0,1,fp8,fp8,0,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,64,0,1,fp8,fp8,0,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,64,128,1,float16,float16,0,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,64,128,1,float16,float16,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,64,0,1,float16,float16,0,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,64,128,1,float16,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,64,128,1,fp8,fp8,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,64,0,1,float16,fp8,0,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,64,128,1,float16,float16,0,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,64,0,1,float16,float16,0,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,64,128,1,float16,fp8,0,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,64,128,1,fp8,fp8,0,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,64,0,1,float16,fp8,0,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,64,0,1,float16,fp8,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,64,0,1,fp8,fp8,0,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,64,0,1,float16,float16,0,0.015583999454975128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,64,128,1,float16,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,64,128,1,fp8,fp8,0,0.021568000316619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,64,0,1,fp8,fp8,0,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,64,128,1,float16,float16,0,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,64,0,1,float16,float16,0,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,64,128,1,float16,fp8,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,64,128,1,fp8,fp8,0,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,64,128,1,fp8,fp8,0,0.02186666677395503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,64,0,1,float16,fp8,0,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,64,0,1,fp8,fp8,0,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,64,128,1,float16,float16,0,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,64,0,1,float16,float16,0,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,64,128,1,float16,fp8,0,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,64,0,1,float16,fp8,0,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,64,0,1,fp8,fp8,0,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,64,0,1,float16,float16,0,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,64,128,1,float16,float16,0,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,64,128,1,float16,fp8,0,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,64,128,1,fp8,fp8,0,0.022117334107557934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,64,0,1,fp8,fp8,0,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,64,0,1,float16,fp8,0,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,64,0,1,float16,float16,0,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,64,128,1,float16,fp8,0,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,64,128,1,float16,float16,0,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,64,128,1,fp8,fp8,0,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,64,0,1,float16,fp8,0,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,64,0,1,float16,fp8,0,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,64,0,1,fp8,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,64,128,1,float16,float16,0,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,64,0,1,float16,float16,0,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,64,128,1,float16,fp8,0,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,64,128,1,fp8,fp8,0,0.022250667214393616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,64,0,1,fp8,fp8,0,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,64,128,1,float16,float16,0,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,64,0,1,float16,float16,0,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,64,128,1,float16,fp8,0,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,64,128,1,fp8,fp8,0,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,64,128,1,fp8,fp8,0,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,64,0,1,float16,fp8,0,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,64,0,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,64,0,1,float16,float16,0,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,64,128,1,float16,float16,0,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,64,128,1,float16,fp8,0,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,64,0,1,float16,float16,0,0.10626666744550069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,64,0,1,float16,fp8,0,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,64,0,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,64,128,1,float16,float16,0,0.10513066252072652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,64,128,1,float16,fp8,0,0.10511466860771179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,64,128,1,fp8,fp8,0,0.1297866702079773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,64,0,1,float16,fp8,0,0.10497599840164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,64,0,1,fp8,fp8,0,0.12793599565823874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,64,128,1,float16,float16,0,0.10899200042088826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,64,0,1,float16,float16,0,0.10896000266075134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,64,128,1,float16,fp8,0,0.10745066404342651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,64,128,1,fp8,fp8,0,0.129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,64,0,1,float16,fp8,0,0.10804266730944316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,64,0,1,fp8,fp8,0,0.12878400087356567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,64,128,1,float16,float16,0,0.11754666765530904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,64,0,1,float16,float16,0,0.11898133158683777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,64,128,1,float16,fp8,0,0.11664000153541565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,64,128,1,fp8,fp8,0,0.1379200021425883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,64,0,1,float16,fp8,0,0.11626133322715759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,64,128,1,float16,float16,0,0.08229866623878479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,64,0,1,fp8,fp8,0,0.13893866539001465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,64,0,1,float16,float16,0,0.08212799827257793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,64,128,1,float16,fp8,0,0.07955199976762135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,64,128,1,fp8,fp8,0,0.08186133205890656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,64,0,1,float16,fp8,0,0.07949333389600118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,64,0,1,fp8,fp8,0,0.0828000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,64,128,1,float16,float16,0,0.057775999108950295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,64,0,1,float16,float16,0,0.057802667220433555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,64,128,1,float16,fp8,0,0.058506667613983154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,64,128,1,fp8,fp8,0,0.07388266424338023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,64,0,1,float16,fp8,0,0.05719999969005585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,64,0,1,fp8,fp8,0,0.07227733234564464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,64,128,1,float16,float16,0,0.05958400170008341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,64,0,1,float16,float16,0,0.059989333152770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,64,128,1,float16,fp8,0,0.059664001067479454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,64,0,1,float16,float16,0,0.06666133304437001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,64,128,1,fp8,fp8,0,0.07369066774845123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,64,0,1,float16,fp8,0,0.05993066728115082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,64,0,1,fp8,fp8,0,0.07380799949169159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,64,128,1,float16,float16,0,0.06703466673692067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,64,128,1,float16,fp8,0,0.06530666848023732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,64,128,1,fp8,fp8,0,0.07734400033950806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,64,0,1,float16,fp8,0,0.06578666468461354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,64,0,1,fp8,fp8,0,0.07860266665617625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,64,128,1,float16,float16,0,0.041690667470296226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,64,0,1,float16,float16,0,0.041840001940727234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,64,128,1,float16,fp8,0,0.03934400031963984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,64,128,1,fp8,fp8,0,0.04970133304595947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,64,0,1,float16,fp8,0,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,64,0,1,fp8,fp8,0,0.05055999755859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,64,128,1,float16,float16,0,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,64,0,1,float16,float16,0,0.033728001018365227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,64,128,1,float16,fp8,0,0.03346666693687439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,64,128,1,fp8,fp8,0,0.042581334710121155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,64,0,1,float16,fp8,0,0.03324799984693527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,64,0,1,fp8,fp8,0,0.04243200023969015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,64,128,1,float16,float16,0,0.034714666505654655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,64,0,1,float16,float16,0,0.03587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,64,128,1,float16,fp8,0,0.034602666894594826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,64,128,1,float16,float16,0,0.036746665835380554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,64,128,1,fp8,fp8,0,0.042837331692377724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,64,0,1,float16,fp8,0,0.034287999073664345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,64,0,1,float16,fp8,0,0.036101333796978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,64,0,1,fp8,fp8,0,0.042677332957585655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,64,0,1,float16,float16,0,0.03629866739114126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,64,128,1,float16,fp8,0,0.03638399889071783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,64,128,1,fp8,fp8,0,0.04664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,64,0,1,fp8,fp8,0,0.04465066889921824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,64,128,1,float16,float16,0,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,64,0,1,float16,float16,0,0.02550933261712392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,64,128,1,float16,fp8,0,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,64,128,1,fp8,fp8,0,0.029872000217437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,64,0,1,float16,fp8,0,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,64,0,1,fp8,fp8,0,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,64,128,1,float16,float16,0,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,64,0,1,float16,float16,0,0.023311999936898548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,64,128,1,float16,fp8,0,0.023957334458827972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,64,128,1,fp8,fp8,0,0.028773332635561626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,64,0,1,float16,fp8,0,0.02365333338578542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,64,0,1,fp8,fp8,0,0.028421332438786823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,64,128,1,float16,float16,0,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,64,0,1,float16,float16,0,0.02362666775782903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,64,128,1,float16,fp8,0,0.02437866727511088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,64,128,1,fp8,fp8,0,0.028714666763941448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,64,0,1,float16,fp8,0,0.02425066630045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,64,0,1,float16,fp8,0,0.02492266645034154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,64,0,1,fp8,fp8,0,0.028864001234372456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,64,128,1,float16,float16,0,0.024645333488782246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,64,0,1,float16,float16,0,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,64,0,1,float16,float16,0,0.024885334074497223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,64,128,1,float16,fp8,0,0.02491733431816101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,64,128,1,fp8,fp8,0,0.029157333076000214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,64,0,1,fp8,fp8,0,0.02972800036271413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,64,128,1,float16,float16,0,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,64,128,1,float16,fp8,0,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,64,128,1,fp8,fp8,0,0.02051199972629547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,64,128,1,fp8,fp8,0,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,64,0,1,float16,fp8,0,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,64,0,1,fp8,fp8,0,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,64,128,1,float16,float16,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,64,0,1,float16,float16,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,64,128,1,float16,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,64,0,1,float16,fp8,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,64,0,1,fp8,fp8,0,0.020261333634455998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,64,128,1,float16,float16,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,64,0,1,float16,float16,0,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,64,128,1,float16,fp8,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,64,128,1,fp8,fp8,0,0.02090666691462199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,64,0,1,float16,fp8,0,0.018288000176350277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,64,0,1,fp8,fp8,0,0.020293333878119785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,64,128,1,float16,float16,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,64,0,1,float16,float16,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,64,128,1,float16,fp8,0,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,64,128,1,fp8,fp8,0,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,64,0,1,float16,fp8,0,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,64,0,1,fp8,fp8,0,0.021802666286627453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,64,128,1,float16,float16,0,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,64,0,1,float16,float16,0,0.014485333114862442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,64,128,1,float16,fp8,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,64,128,1,fp8,fp8,0,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,64,0,1,float16,fp8,0,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,64,0,1,float16,fp8,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,64,0,1,fp8,fp8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,64,128,1,float16,float16,0,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,64,0,1,float16,float16,0,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,64,128,1,float16,fp8,0,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,64,128,1,fp8,fp8,0,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,64,0,1,fp8,fp8,0,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,64,128,1,float16,float16,0,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,64,0,1,fp8,fp8,0,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,64,0,1,float16,float16,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,64,0,1,float16,float16,0,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,64,128,1,float16,fp8,0,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,64,128,1,fp8,fp8,0,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,64,0,1,float16,fp8,0,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,64,128,1,float16,float16,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,64,128,1,float16,fp8,0,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,64,128,1,fp8,fp8,0,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,64,0,1,float16,fp8,0,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,64,0,1,fp8,fp8,0,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,64,128,1,float16,float16,0,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,64,0,1,float16,float16,0,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,64,128,1,float16,fp8,0,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,64,128,1,fp8,fp8,0,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,64,0,1,float16,fp8,0,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,64,0,1,fp8,fp8,0,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,64,128,1,float16,float16,0,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,64,0,1,fp8,fp8,0,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,64,0,1,float16,float16,0,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,64,0,1,float16,float16,0,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,64,128,1,float16,fp8,0,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,64,128,1,fp8,fp8,0,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,64,128,1,fp8,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,64,0,1,float16,fp8,0,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,64,128,1,float16,float16,0,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,64,128,1,float16,fp8,0,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,64,0,1,float16,fp8,0,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,64,0,1,fp8,fp8,0,0.01836266616980235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,64,128,1,float16,float16,0,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,64,0,1,float16,float16,0,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,64,128,1,float16,fp8,0,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,64,128,1,fp8,fp8,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,64,0,1,float16,fp8,0,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,64,0,1,fp8,fp8,0,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,64,128,1,float16,float16,0,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,64,0,1,float16,float16,0,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,64,128,1,float16,fp8,0,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,64,128,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,64,0,1,float16,fp8,0,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,64,0,1,fp8,fp8,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,64,128,1,float16,float16,0,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,64,0,1,float16,float16,0,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,64,128,1,float16,fp8,0,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,64,128,1,fp8,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,64,0,1,float16,fp8,0,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,64,0,1,fp8,fp8,0,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,64,128,1,float16,float16,0,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,64,0,1,float16,float16,0,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,64,128,1,float16,fp8,0,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,64,128,1,fp8,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,64,0,1,float16,fp8,0,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,64,0,1,fp8,fp8,0,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,64,128,1,float16,float16,0,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,64,0,1,float16,float16,0,0.012608000387748083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,64,128,1,float16,fp8,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,64,128,1,fp8,fp8,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,64,0,1,float16,fp8,0,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,64,0,1,fp8,fp8,0,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,64,128,1,float16,float16,0,0.012378666549921036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,64,128,1,float16,float16,0,0.012527999778588613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,64,0,1,float16,float16,0,0.01250133290886879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,64,128,1,float16,fp8,0,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,64,128,1,fp8,fp8,0,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,64,0,1,float16,fp8,0,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,64,0,1,fp8,fp8,0,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,64,0,1,float16,float16,0,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,64,128,1,float16,fp8,0,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,64,128,1,fp8,fp8,0,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,64,0,1,float16,fp8,0,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,64,0,1,fp8,fp8,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,64,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,64,128,1,float16,float16,0,0.012495999534924826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,64,0,1,float16,float16,0,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,64,128,1,float16,fp8,0,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,64,128,1,fp8,fp8,0,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,64,0,1,float16,fp8,0,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,64,128,1,float16,float16,0,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,64,0,1,float16,float16,0,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,64,128,1,float16,fp8,0,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,64,128,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,64,0,1,float16,fp8,0,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,64,0,1,float16,fp8,0,0.05486933390299479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,64,0,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,64,128,1,float16,float16,0,0.05532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,64,0,1,float16,float16,0,0.05508799850940704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,64,128,1,float16,fp8,0,0.05602666735649109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,64,128,1,float16,fp8,0,0.05481066803137461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,64,128,1,fp8,fp8,0,0.09456533193588257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,64,0,1,fp8,fp8,0,0.09427733222643535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,64,128,1,float16,float16,0,0.056320001681645714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,64,0,1,float16,float16,0,0.05665066838264465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,64,128,1,fp8,fp8,0,0.09546132882436116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,64,0,1,float16,fp8,0,0.05625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,64,0,1,fp8,fp8,0,0.09636800487836202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,64,128,1,float16,float16,0,0.06238933404286703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,64,0,1,float16,float16,0,0.06341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,64,128,1,float16,fp8,0,0.061493332187334694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,64,128,1,fp8,fp8,0,0.09984532992045085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,64,0,1,float16,fp8,0,0.06132266422112783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,64,0,1,fp8,fp8,0,0.0997226635615031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,64,128,1,float16,float16,0,0.03900266687075297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,64,0,1,float16,float16,0,0.039120001097520195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,64,128,1,float16,fp8,0,0.037690666814645134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,64,128,1,fp8,fp8,0,0.06050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,64,0,1,float16,fp8,0,0.03718400001525879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,64,0,1,fp8,fp8,0,0.060378665725390114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,64,128,1,float16,float16,0,0.0322080006202062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,64,0,1,float16,float16,0,0.03219733387231827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,64,128,1,float16,fp8,0,0.03180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,64,128,1,fp8,fp8,0,0.05304533243179321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,64,0,1,float16,fp8,0,0.03216533362865448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,64,0,1,fp8,fp8,0,0.053247998158137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,64,128,1,float16,float16,0,0.032501332461833954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,64,0,1,float16,float16,0,0.033215999603271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,64,128,1,float16,fp8,0,0.032629333436489105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,64,128,1,fp8,fp8,0,0.05395199855168661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,64,0,1,float16,fp8,0,0.03226666649182638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,64,0,1,fp8,fp8,0,0.05343466500441233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,64,128,1,float16,float16,0,0.03432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,64,0,1,float16,float16,0,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,64,128,1,float16,fp8,0,0.03452266752719879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,64,128,1,fp8,fp8,0,0.05780800183614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,64,0,1,float16,fp8,0,0.03432533393303553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,64,0,1,fp8,fp8,0,0.05657066901524862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,64,128,1,float16,float16,0,0.022570667167504627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,64,0,1,float16,float16,0,0.022778667509555817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,64,128,1,float16,fp8,0,0.02242133269707362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,64,128,1,fp8,fp8,0,0.035029334326585136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,64,128,1,float16,fp8,0,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,64,0,1,float16,fp8,0,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,64,0,1,fp8,fp8,0,0.0342399999499321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,64,128,1,float16,float16,0,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,64,0,1,float16,float16,0,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,64,128,1,fp8,fp8,0,0.033146666983763375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,64,0,1,float16,fp8,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,64,0,1,fp8,fp8,0,0.033013333876927696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,64,128,1,float16,float16,0,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,64,0,1,float16,float16,0,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,64,128,1,float16,fp8,0,0.021514666577180225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,64,128,1,fp8,fp8,0,0.034458667039871216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,64,0,1,float16,fp8,0,0.021903999149799347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,64,0,1,fp8,fp8,0,0.03380800038576126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,64,128,1,float16,float16,0,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,64,0,1,float16,float16,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,64,128,1,float16,fp8,0,0.022656001150608063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,64,128,1,fp8,fp8,0,0.034490667283535004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,64,0,1,float16,fp8,0,0.022261333962281544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,64,128,1,fp8,fp8,0,0.02483733246723811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,64,0,1,fp8,fp8,0,0.03405333310365677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,64,128,1,float16,float16,0,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,64,0,1,float16,float16,0,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,64,128,1,float16,fp8,0,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,64,0,1,float16,fp8,0,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,64,0,1,fp8,fp8,0,0.024527999262015026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,64,128,1,float16,float16,0,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,64,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,64,128,1,float16,fp8,0,0.01674666628241539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,64,128,1,fp8,fp8,0,0.023647998770078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,64,0,1,float16,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,64,0,1,fp8,fp8,0,0.023973333338896435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,64,128,1,float16,float16,0,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,64,0,1,float16,float16,0,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,64,128,1,float16,fp8,0,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,64,128,1,fp8,fp8,0,0.024112001061439514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,64,0,1,float16,fp8,0,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,64,0,1,fp8,fp8,0,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,64,128,1,float16,float16,0,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,64,0,1,float16,float16,0,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,64,0,1,float16,float16,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,64,128,1,float16,fp8,0,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,64,128,1,fp8,fp8,0,0.02430933217207591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,64,0,1,float16,fp8,0,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,64,0,1,fp8,fp8,0,0.01959466685851415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,64,0,1,fp8,fp8,0,0.024746666351954143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,64,128,1,float16,float16,0,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,64,128,1,float16,fp8,0,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,64,128,1,fp8,fp8,0,0.01979200045267741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,64,0,1,float16,fp8,0,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,64,128,1,float16,float16,0,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,64,0,1,float16,float16,0,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,64,128,1,float16,fp8,0,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,64,128,1,fp8,fp8,0,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,64,0,1,float16,fp8,0,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,64,0,1,fp8,fp8,0,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,64,128,1,float16,float16,0,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,64,0,1,float16,float16,0,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,64,128,1,float16,fp8,0,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,64,128,1,fp8,fp8,0,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,64,0,1,float16,fp8,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,64,0,1,fp8,fp8,0,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,64,128,1,float16,float16,0,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,64,0,1,float16,float16,0,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,64,128,1,float16,fp8,0,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,64,128,1,fp8,fp8,0,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,64,0,1,float16,fp8,0,0.01369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,64,0,1,fp8,fp8,0,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,64,0,1,fp8,fp8,0,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,64,128,1,float16,float16,0,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,64,0,1,float16,float16,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,64,128,1,float16,fp8,0,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,64,128,1,fp8,fp8,0,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,64,0,1,float16,fp8,0,0.012554666648308435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,64,128,1,float16,float16,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,64,0,1,float16,float16,0,0.01239466667175293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,64,128,1,float16,fp8,0,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,64,128,1,fp8,fp8,0,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,64,0,1,float16,fp8,0,0.01246400053302447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,64,0,1,fp8,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,64,128,1,float16,float16,0,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,64,0,1,float16,float16,0,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,64,128,1,float16,fp8,0,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,64,128,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,64,0,1,float16,fp8,0,0.012469333906968435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,64,128,1,fp8,fp8,0,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,64,0,1,fp8,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,64,128,1,float16,float16,0,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,64,0,1,float16,float16,0,0.012202666451533636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,64,128,1,float16,fp8,0,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,64,0,1,float16,fp8,0,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,64,128,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,64,0,1,fp8,fp8,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,64,128,1,float16,float16,0,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,64,0,1,float16,float16,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,64,128,1,float16,fp8,0,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,64,0,1,float16,fp8,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,64,0,1,fp8,fp8,0,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,64,128,1,float16,float16,0,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,64,0,1,float16,float16,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,64,128,1,float16,fp8,0,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,64,128,1,fp8,fp8,0,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,64,128,1,float16,fp8,0,0.012341332932313284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,64,0,1,float16,fp8,0,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,64,0,1,fp8,fp8,0,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,64,128,1,float16,float16,0,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,64,0,1,float16,float16,0,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,64,128,1,fp8,fp8,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,64,0,1,float16,fp8,0,0.012223999947309494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,64,0,1,fp8,fp8,0,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,64,128,1,float16,float16,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,64,0,1,float16,float16,0,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,64,128,1,float16,fp8,0,0.012383999923865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,64,128,1,float16,fp8,0,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,64,128,1,fp8,fp8,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,64,0,1,float16,fp8,0,0.012298667182525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,64,0,1,fp8,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,64,128,1,float16,float16,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,64,0,1,float16,float16,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,64,128,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,64,128,1,fp8,fp8,0,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,64,0,1,float16,fp8,0,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,64,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,64,128,1,float16,float16,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,64,0,1,float16,float16,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,64,128,1,float16,fp8,0,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,64,0,1,float16,fp8,0,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,64,0,1,float16,fp8,0,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,64,0,1,fp8,fp8,0,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,64,0,1,fp8,fp8,0,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,64,128,1,float16,float16,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,64,0,1,float16,float16,0,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,64,128,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,64,0,1,float16,fp8,0,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,64,128,1,fp8,fp8,0,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,64,128,1,float16,float16,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,64,0,1,float16,float16,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,64,128,1,float16,fp8,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,64,128,1,fp8,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,64,128,1,fp8,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,64,0,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,64,128,1,float16,float16,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,64,0,1,float16,float16,0,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,64,128,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,64,0,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,64,0,1,fp8,fp8,0,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,64,128,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,64,0,1,float16,float16,0,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,64,128,1,float16,fp8,0,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,64,128,1,fp8,fp8,0,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,64,0,1,float16,fp8,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,64,0,1,fp8,fp8,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,64,0,1,fp8,fp8,0,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,64,128,1,float16,float16,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,64,0,1,float16,float16,0,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,64,128,1,float16,fp8,0,0.012240000069141388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,64,128,1,fp8,fp8,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,64,0,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,64,128,1,float16,float16,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,64,0,1,float16,float16,0,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,64,128,1,float16,fp8,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,64,0,1,float16,float16,0,0.033557333052158356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,64,128,1,fp8,fp8,0,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,64,0,1,float16,fp8,0,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,64,0,1,float16,fp8,0,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,64,0,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,64,128,1,float16,float16,0,0.033674667278925575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,64,128,1,float16,fp8,0,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,64,128,1,fp8,fp8,0,0.077824001510938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,64,0,1,fp8,fp8,0,0.07700799902280171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,64,128,1,float16,float16,0,0.034485332667827606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,64,0,1,float16,float16,0,0.034832000732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,64,128,1,float16,fp8,0,0.03435199956099192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,64,128,1,fp8,fp8,0,0.07811200122038524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,64,0,1,float16,fp8,0,0.033999999364217125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,64,0,1,fp8,fp8,0,0.07785599927107494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,64,128,1,float16,float16,0,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,64,0,1,float16,float16,0,0.036277333895365395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,64,128,1,float16,fp8,0,0.03602666656176249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,64,0,1,float16,fp8,0,0.035887998839219414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,64,128,1,fp8,fp8,0,0.08059733112653096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,64,0,1,fp8,fp8,0,0.08083199958006541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,64,128,1,float16,float16,0,0.022848000129063923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,64,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,64,128,1,float16,fp8,0,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,64,128,1,fp8,fp8,0,0.04755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,64,0,1,float16,fp8,0,0.0229066660006841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,64,0,1,fp8,fp8,0,0.04788800080617269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,64,128,1,float16,float16,0,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,64,0,1,float16,float16,0,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,64,128,1,float16,fp8,0,0.02195200075705846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,64,128,1,fp8,fp8,0,0.04718933502833048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,64,0,1,float16,fp8,0,0.02197866638501485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,64,0,1,fp8,fp8,0,0.04680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,64,0,1,float16,fp8,0,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,64,128,1,float16,float16,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,64,0,1,float16,float16,0,0.022458667556444805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,64,128,1,float16,fp8,0,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,64,128,1,fp8,fp8,0,0.047040000557899475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,64,0,1,fp8,fp8,0,0.04742933313051859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,64,128,1,float16,float16,0,0.02288000037272771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,64,0,1,float16,float16,0,0.022858666876951855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,64,0,1,float16,float16,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,64,128,1,float16,fp8,0,0.022730665902296703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,64,128,1,fp8,fp8,0,0.047594666481018066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,64,0,1,float16,fp8,0,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,64,0,1,fp8,fp8,0,0.030666666726271313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,64,0,1,fp8,fp8,0,0.04747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,64,128,1,float16,float16,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,64,128,1,float16,fp8,0,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,64,128,1,fp8,fp8,0,0.030986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,64,0,1,float16,fp8,0,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,64,0,1,fp8,fp8,0,0.029968000948429108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,64,128,1,float16,float16,0,0.015637333194414776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,64,0,1,float16,float16,0,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,64,128,1,float16,fp8,0,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,64,128,1,fp8,fp8,0,0.029685333371162415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,64,0,1,float16,fp8,0,0.015861333658297855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,64,0,1,fp8,fp8,0,0.030031998952229817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,64,128,1,float16,float16,0,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,64,0,1,float16,float16,0,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,64,128,1,float16,fp8,0,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,64,128,1,fp8,fp8,0,0.03070399910211563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,64,128,1,fp8,fp8,0,0.030016000072161358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,64,0,1,float16,fp8,0,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,64,128,1,float16,float16,0,0.015813333292802174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,64,0,1,float16,float16,0,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,64,128,1,float16,fp8,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,64,0,1,float16,fp8,0,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,64,0,1,fp8,fp8,0,0.02266666789849599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,64,0,1,fp8,fp8,0,0.03075733284155528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,64,128,1,float16,float16,0,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,64,0,1,float16,float16,0,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,64,128,1,float16,fp8,0,0.013690666606028875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,64,128,1,fp8,fp8,0,0.022783999641736347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,64,0,1,float16,fp8,0,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,64,128,1,float16,float16,0,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,64,128,1,float16,float16,0,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,64,0,1,float16,float16,0,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,64,128,1,fp8,fp8,0,0.02266666789849599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,64,128,1,float16,fp8,0,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,64,128,1,fp8,fp8,0,0.022522665560245514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,64,0,1,float16,fp8,0,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,64,0,1,fp8,fp8,0,0.022053333620230358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,64,0,1,float16,float16,0,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,64,128,1,float16,fp8,0,0.01369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,64,0,1,float16,fp8,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,64,0,1,fp8,fp8,0,0.0225600004196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,64,128,1,float16,float16,0,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,64,0,1,float16,float16,0,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,64,128,1,float16,fp8,0,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,64,128,1,fp8,fp8,0,0.022117334107557934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,64,0,1,float16,fp8,0,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,64,0,1,fp8,fp8,0,0.022810667753219604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,64,128,1,float16,float16,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,64,0,1,float16,float16,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,64,0,1,float16,float16,0,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,64,128,1,float16,fp8,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,64,128,1,fp8,fp8,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,64,0,1,float16,fp8,0,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,64,0,1,fp8,fp8,0,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,64,128,1,float16,float16,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,64,128,1,float16,fp8,0,0.012175999581813812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,64,128,1,fp8,fp8,0,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,64,0,1,float16,fp8,0,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,64,0,1,fp8,fp8,0,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,64,128,1,float16,float16,0,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,64,0,1,float16,float16,0,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,64,128,1,float16,fp8,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,64,128,1,fp8,fp8,0,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,64,0,1,float16,fp8,0,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,64,0,1,fp8,fp8,0,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,64,128,1,float16,float16,0,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,64,0,1,float16,float16,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,64,128,1,float16,fp8,0,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,64,128,1,fp8,fp8,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,64,0,1,float16,fp8,0,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,64,0,1,fp8,fp8,0,0.01836266616980235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,64,128,1,float16,float16,0,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,64,0,1,float16,float16,0,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,64,128,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,64,128,1,fp8,fp8,0,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,64,0,1,float16,fp8,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,64,0,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,64,128,1,float16,float16,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,64,0,1,float16,float16,0,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,64,128,1,float16,fp8,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,64,128,1,fp8,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,64,0,1,float16,fp8,0,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,64,0,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,64,128,1,float16,float16,0,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,64,0,1,float16,float16,0,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,64,128,1,float16,fp8,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,64,128,1,fp8,fp8,0,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,64,0,1,float16,fp8,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,64,0,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,64,128,1,float16,float16,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,64,0,1,float16,float16,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,64,128,1,float16,fp8,0,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,64,128,1,fp8,fp8,0,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,64,0,1,float16,fp8,0,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,64,0,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,64,128,1,float16,float16,0,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,64,0,1,float16,float16,0,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,64,128,1,float16,fp8,0,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,64,128,1,fp8,fp8,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,64,0,1,float16,fp8,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,64,0,1,fp8,fp8,0,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,64,128,1,float16,float16,0,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,64,0,1,float16,float16,0,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,64,128,1,float16,fp8,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,64,128,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,64,0,1,float16,fp8,0,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,64,0,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,64,128,1,float16,float16,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,64,0,1,float16,float16,0,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,64,128,1,float16,fp8,0,0.011920000116030375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,64,128,1,fp8,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,64,0,1,float16,fp8,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,64,0,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,64,128,1,float16,float16,0,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,64,128,1,float16,float16,0,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,64,0,1,float16,float16,0,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,64,0,1,float16,float16,0,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,64,128,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,64,128,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,64,0,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,64,0,1,fp8,fp8,0,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,64,128,1,float16,fp8,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,64,128,1,fp8,fp8,0,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,64,0,1,float16,fp8,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,64,0,1,fp8,fp8,0,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,64,128,1,float16,float16,0,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,64,0,1,float16,float16,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,64,128,1,float16,fp8,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,64,128,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,64,0,1,float16,fp8,0,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,64,0,1,fp8,fp8,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,64,128,1,float16,float16,0,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,64,0,1,float16,float16,0,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,64,128,1,float16,fp8,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,64,128,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,64,0,1,float16,fp8,0,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,64,0,1,fp8,fp8,0,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,64,128,1,float16,float16,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,64,0,1,float16,float16,0,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,64,128,1,float16,fp8,0,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,64,128,1,fp8,fp8,0,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,64,0,1,float16,fp8,0,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,64,0,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,64,128,1,float16,float16,0,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,64,0,1,float16,float16,0,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,64,128,1,float16,fp8,0,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,64,128,1,fp8,fp8,0,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,64,128,1,fp8,fp8,0,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,64,0,1,float16,fp8,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,64,0,1,fp8,fp8,0,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,64,128,1,float16,float16,0,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,64,0,1,float16,float16,0,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,64,128,1,float16,fp8,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,64,0,1,float16,fp8,0,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,64,0,1,fp8,fp8,0,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,64,128,1,float16,float16,0,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,64,0,1,float16,float16,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,64,128,1,float16,fp8,0,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,64,128,1,fp8,fp8,0,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,64,0,1,float16,fp8,0,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,64,0,1,fp8,fp8,0,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,64,128,1,float16,float16,0,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,64,0,1,float16,float16,0,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,64,128,1,float16,fp8,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,64,128,1,fp8,fp8,0,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,64,0,1,float16,fp8,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,64,0,1,fp8,fp8,0,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,64,128,1,float16,float16,0,0.0268053337931633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,64,0,1,float16,float16,0,0.02698666602373123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,64,128,1,float16,fp8,0,0.026901334524154663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,64,128,1,fp8,fp8,0,0.07022400200366974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,64,128,1,float16,fp8,0,0.02771199991305669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,64,128,1,fp8,fp8,0,0.07040533423423767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,64,0,1,float16,fp8,0,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,64,0,1,fp8,fp8,0,0.07135466734568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,64,128,1,float16,float16,0,0.027488000690937042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,64,0,1,float16,float16,0,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,64,0,1,float16,fp8,0,0.027290667096773785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,64,0,1,fp8,fp8,0,0.07086400190989177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,64,128,1,float16,float16,0,0.02809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,64,0,1,float16,float16,0,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,64,128,1,float16,fp8,0,0.028373333315054577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,64,128,1,fp8,fp8,0,0.07214933137098949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,64,0,1,float16,fp8,0,0.02809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,64,128,1,float16,float16,0,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,64,0,1,float16,fp8,0,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,64,0,1,fp8,fp8,0,0.07099733253320058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,64,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,64,128,1,float16,fp8,0,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,64,128,1,fp8,fp8,0,0.043605332573254905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,64,0,1,fp8,fp8,0,0.044256001710891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,64,128,1,float16,float16,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,64,0,1,float16,float16,0,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,64,128,1,float16,fp8,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,64,128,1,fp8,fp8,0,0.04349866509437561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,64,0,1,float16,fp8,0,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,64,0,1,fp8,fp8,0,0.043141335248947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,64,128,1,float16,float16,0,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,64,0,1,float16,float16,0,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,64,128,1,float16,fp8,0,0.018485333770513535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,64,128,1,fp8,fp8,0,0.043023998538653054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,64,0,1,float16,fp8,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,64,0,1,fp8,fp8,0,0.04353600243727366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,64,128,1,float16,float16,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,64,0,1,float16,float16,0,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,64,0,1,float16,float16,0,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,64,128,1,float16,fp8,0,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,64,128,1,fp8,fp8,0,0.04378133515516917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,64,128,1,fp8,fp8,0,0.028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,64,0,1,fp8,fp8,0,0.02900800108909607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,64,0,1,float16,fp8,0,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,64,128,1,float16,float16,0,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,64,0,1,fp8,fp8,0,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,64,128,1,float16,fp8,0,0.014085333794355392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,64,0,1,float16,fp8,0,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,64,128,1,float16,float16,0,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,64,0,1,float16,float16,0,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,64,128,1,float16,fp8,0,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,64,128,1,fp8,fp8,0,0.028501334289709728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,64,0,1,float16,fp8,0,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,64,0,1,fp8,fp8,0,0.028565332293510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,64,128,1,float16,float16,0,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,64,0,1,float16,float16,0,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,64,128,1,float16,fp8,0,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,64,128,1,fp8,fp8,0,0.02831999957561493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,64,0,1,float16,fp8,0,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,64,0,1,fp8,fp8,0,0.028965334097544353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,64,128,1,float16,float16,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,64,0,1,float16,float16,0,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,64,128,1,float16,fp8,0,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,64,128,1,fp8,fp8,0,0.028880000114440918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,64,0,1,float16,fp8,0,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,64,0,1,fp8,fp8,0,0.028618666032950085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,64,0,1,fp8,fp8,0,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,64,128,1,float16,float16,0,0.012421333541472753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,64,0,1,float16,float16,0,0.012272000312805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,64,128,1,float16,fp8,0,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,64,128,1,fp8,fp8,0,0.02163733293612798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,64,0,1,float16,fp8,0,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,64,128,1,float16,float16,0,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,64,0,1,float16,float16,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,64,128,1,float16,fp8,0,0.012453333785136541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,64,128,1,fp8,fp8,0,0.0215786670645078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,64,0,1,float16,fp8,0,0.012666666259368261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,64,0,1,fp8,fp8,0,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,64,128,1,float16,float16,0,0.012442667037248611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,64,0,1,float16,float16,0,0.012522666404644648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,64,128,1,float16,fp8,0,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,64,128,1,fp8,fp8,0,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,64,0,1,float16,fp8,0,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,64,0,1,fp8,fp8,0,0.02163200080394745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,64,128,1,float16,float16,0,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,64,0,1,float16,float16,0,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,64,128,1,float16,fp8,0,0.012421333541472753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,64,128,1,fp8,fp8,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,64,0,1,float16,fp8,0,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,64,0,1,fp8,fp8,0,0.021701333423455555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,64,128,1,float16,float16,0,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,64,0,1,float16,float16,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,64,128,1,float16,fp8,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,64,128,1,fp8,fp8,0,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,64,128,1,fp8,fp8,0,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,64,0,1,float16,fp8,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,64,0,1,fp8,fp8,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,64,128,1,float16,float16,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,64,0,1,float16,float16,0,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,64,128,1,float16,fp8,0,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,64,0,1,float16,fp8,0,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,64,0,1,fp8,fp8,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,64,0,1,float16,fp8,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,64,128,1,float16,float16,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,64,0,1,float16,float16,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,64,128,1,float16,fp8,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,64,128,1,fp8,fp8,0,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,64,0,1,fp8,fp8,0,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,64,128,1,float16,float16,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,64,0,1,float16,float16,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,64,128,1,float16,fp8,0,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,64,128,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,64,0,1,float16,fp8,0,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,64,0,1,fp8,fp8,0,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,64,128,1,float16,float16,0,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,64,0,1,float16,float16,0,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,64,128,1,float16,fp8,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,64,128,1,fp8,fp8,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,64,0,1,float16,fp8,0,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,64,0,1,fp8,fp8,0,0.018250666558742523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,64,128,1,float16,float16,0,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,64,0,1,float16,float16,0,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,64,128,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,64,128,1,fp8,fp8,0,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,64,0,1,float16,fp8,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,64,0,1,fp8,fp8,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,64,128,1,float16,float16,0,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,64,0,1,float16,float16,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,64,128,1,float16,fp8,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,64,128,1,fp8,fp8,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,64,0,1,float16,fp8,0,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,64,0,1,fp8,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,64,128,1,float16,float16,0,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,64,0,1,float16,float16,0,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,64,128,1,float16,fp8,0,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,64,128,1,fp8,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,64,128,1,fp8,fp8,0,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,64,0,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,64,0,1,fp8,fp8,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,64,128,1,float16,float16,0,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,64,0,1,float16,float16,0,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,64,128,1,float16,fp8,0,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,64,128,1,fp8,fp8,0,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,64,0,1,float16,fp8,0,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,64,0,1,fp8,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,64,128,1,float16,float16,0,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,64,0,1,float16,float16,0,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,64,128,1,float16,fp8,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,64,0,1,float16,fp8,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,64,0,1,fp8,fp8,0,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,64,0,1,fp8,fp8,0,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,64,128,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,64,0,1,float16,float16,0,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,64,128,1,float16,fp8,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,64,128,1,fp8,fp8,0,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,64,0,1,float16,fp8,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,64,128,1,float16,float16,0,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,64,0,1,float16,float16,0,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,64,128,1,float16,fp8,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,64,128,1,fp8,fp8,0,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,64,0,1,float16,fp8,0,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,64,0,1,fp8,fp8,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,64,128,1,float16,float16,0,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,64,0,1,float16,float16,0,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,64,128,1,float16,fp8,0,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,64,128,1,fp8,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,64,0,1,float16,fp8,0,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,64,0,1,fp8,fp8,0,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,64,128,1,float16,float16,0,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,64,0,1,float16,float16,0,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,64,128,1,float16,fp8,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,64,128,1,fp8,fp8,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,64,0,1,float16,fp8,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,64,0,1,fp8,fp8,0,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,64,128,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,64,128,1,float16,float16,0,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,64,0,1,float16,float16,0,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,64,128,1,float16,fp8,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,64,128,1,fp8,fp8,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,64,0,1,float16,fp8,0,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,64,0,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,64,0,1,float16,float16,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,64,128,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,64,128,1,fp8,fp8,0,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,64,0,1,float16,fp8,0,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,64,0,1,fp8,fp8,0,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,64,128,1,float16,float16,0,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,64,0,1,float16,float16,0,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,64,128,1,float16,fp8,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,64,128,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,64,0,1,float16,fp8,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,64,0,1,fp8,fp8,0,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,64,128,1,float16,float16,0,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,64,0,1,float16,float16,0,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,64,128,1,float16,fp8,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,64,128,1,fp8,fp8,0,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,64,0,1,float16,fp8,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,64,0,1,fp8,fp8,0,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,64,128,1,float16,float16,0,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,64,0,1,float16,float16,0,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,64,128,1,float16,fp8,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,64,128,1,fp8,fp8,0,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,64,0,1,float16,fp8,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,64,0,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,64,128,1,float16,float16,0,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,64,0,1,float16,float16,0,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,64,128,1,float16,fp8,0,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,64,128,1,fp8,fp8,0,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,64,0,1,float16,fp8,0,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,64,128,1,float16,float16,0,0.39581334590911865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,64,0,1,fp8,fp8,0,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,64,128,1,float16,fp8,0,0.3921866814295451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,64,128,1,fp8,fp8,0,0.5066666603088379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,64,128,1,float16,float16,0,0.42265065511067706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,64,128,1,float16,fp8,0,0.41654932498931885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,64,128,1,fp8,fp8,0,0.5328213373819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,64,0,1,float16,float16,0,2.4957547187805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,64,128,1,float16,float16,0,0.23668267329533896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,64,0,1,float16,fp8,0,2.449087937672933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,64,0,1,fp8,fp8,0,2.251178741455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,64,0,1,float16,float16,0,2.5033013025919595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,64,128,1,float16,fp8,0,0.2327359914779663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,64,128,1,fp8,fp8,0,0.3041013280550639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,64,128,1,float16,float16,0,0.20928533871968588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,64,0,1,float16,fp8,0,2.4871840476989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,64,0,1,float16,float16,0,1.290608008702596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,64,0,1,fp8,fp8,0,2.261045296986898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,64,0,1,fp8,fp8,0,1.1161279678344727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,64,128,1,float16,fp8,0,0.20443199078241983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,64,0,1,float16,fp8,0,1.2862293720245361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,64,128,1,fp8,fp8,0,0.2814133365948995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,64,128,1,float16,float16,0,0.22431466976801553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,64,0,1,float16,float16,0,1.2633386452992756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,64,128,1,fp8,fp8,0,0.2941280007362366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,64,128,1,float16,fp8,0,0.22052266200383505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,64,0,1,float16,fp8,0,1.2861066659291585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,64,0,1,fp8,fp8,0,1.0963093439737956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,64,128,1,float16,float16,0,0.13222400347391763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,64,0,1,float16,float16,0,1.2812906901041667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,64,128,1,float16,fp8,0,0.1318186620871226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,64,0,1,float16,float16,0,0.6259093284606934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,64,0,1,float16,fp8,0,1.2779253323872883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,64,128,1,fp8,fp8,0,0.1737013260523478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,64,0,1,fp8,fp8,0,1.1051146984100342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,64,128,1,float16,float16,0,0.12158933281898499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,64,0,1,float16,fp8,0,0.6313440004984537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,64,0,1,fp8,fp8,0,0.5783146619796753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,64,128,1,float16,fp8,0,0.12302933136622111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,64,128,1,fp8,fp8,0,0.16080533464749655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,64,0,1,float16,float16,0,0.6115573247273763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,64,128,1,float16,float16,0,0.12578133742014566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,64,0,1,float16,fp8,0,0.6132746537526449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,64,128,1,float16,fp8,0,0.1262399951616923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,64,0,1,fp8,fp8,0,0.5751253366470337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,64,128,1,fp8,fp8,0,0.17142399152119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,64,0,1,float16,float16,0,0.6269013484319051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,64,128,1,float16,float16,0,0.09309333562850952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,64,0,1,float16,fp8,0,0.6122986475626627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,64,0,1,fp8,fp8,0,0.570522665977478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,64,128,1,float16,fp8,0,0.09259733557701111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,64,0,1,float16,fp8,0,0.3380213181177775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,64,0,1,float16,float16,0,0.33691199620564777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,64,128,1,fp8,fp8,0,0.11778133114178975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,64,128,1,float16,float16,0,0.08943999807039897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,64,0,1,fp8,fp8,0,0.31361599763234455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,64,128,1,float16,fp8,0,0.08904000123341878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,64,0,1,float16,float16,0,0.3311733404795329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,64,0,1,fp8,fp8,0,0.31033066908518475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,64,128,1,fp8,fp8,0,0.10921600461006165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,64,128,1,fp8,fp8,0,0.10708266496658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,64,0,1,float16,fp8,0,0.3356746832529704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,64,0,1,fp8,fp8,0,0.31195199489593506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,64,0,1,float16,fp8,0,0.3323626716931661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,64,128,1,float16,float16,0,0.09101866682370503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,64,128,1,float16,fp8,0,0.09222933650016785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,64,0,1,float16,float16,0,0.3394240140914917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,64,128,1,float16,float16,0,0.30261866251627606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,64,128,1,float16,fp8,0,0.2986239989598592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,64,128,1,fp8,fp8,0,0.38926398754119873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,64,0,1,float16,float16,0,1.4433813095092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,64,128,1,float16,float16,0,0.3234933416048686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,64,0,1,float16,fp8,0,1.4497653643290203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,64,128,1,float16,fp8,0,0.31942399342854816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,64,0,1,fp8,fp8,0,1.2670400142669678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,64,128,1,fp8,fp8,0,0.41044267018636066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,64,128,1,float16,float16,0,0.1818079948425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,64,128,1,float16,fp8,0,0.1784320076306661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,64,0,1,float16,float16,0,1.4651093482971191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,64,128,1,fp8,fp8,0,0.23402667045593262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,64,0,1,float16,float16,0,0.7243680159250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,64,128,1,float16,float16,0,0.1600320041179657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,64,0,1,fp8,fp8,0,1.286511977513631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,64,0,1,float16,fp8,0,1.4576212565104167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,64,0,1,float16,fp8,0,0.7166399955749512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,64,0,1,fp8,fp8,0,0.6655360062917074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,64,128,1,float16,fp8,0,0.15713600317637125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,64,128,1,fp8,fp8,0,0.21714133024215698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,64,0,1,float16,float16,0,0.7012693087259928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,64,128,1,float16,float16,0,0.16782933473587036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,64,0,1,float16,fp8,0,0.7034506797790527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,64,0,1,fp8,fp8,0,0.6454453468322754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,64,0,1,float16,float16,0,0.707360029220581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,64,128,1,float16,fp8,0,0.1646933356920878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,64,128,1,fp8,fp8,0,0.22633065780003866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,64,128,1,float16,float16,0,0.10321066776911418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,64,0,1,float16,fp8,0,0.7127679983774821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,64,128,1,float16,fp8,0,0.10340266426404317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,64,128,1,fp8,fp8,0,0.1376213332017263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,64,0,1,float16,float16,0,0.36746664841969806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,64,0,1,fp8,fp8,0,0.6606719891230265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,64,128,1,float16,float16,0,0.09575999776522319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,64,0,1,float16,fp8,0,0.3702293237050374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,64,0,1,fp8,fp8,0,0.34963734944661456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,64,128,1,float16,fp8,0,0.09532266855239868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,64,128,1,float16,float16,0,0.09712533156077068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,64,0,1,float16,fp8,0,0.3768746852874756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,64,0,1,fp8,fp8,0,0.34014399846394855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,64,0,1,float16,float16,0,0.3641759951909383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,64,128,1,fp8,fp8,0,0.12865599989891052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,64,128,1,float16,fp8,0,0.09745066364606221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,64,0,1,float16,float16,0,0.367029349009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,64,128,1,fp8,fp8,0,0.13319999972979227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,64,0,1,fp8,fp8,0,0.3441760142644246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,64,0,1,float16,fp8,0,0.37754666805267334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,64,128,1,float16,float16,0,0.07309866448243459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,64,0,1,float16,float16,0,0.21845867236455283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,64,128,1,float16,fp8,0,0.0746559997399648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,64,128,1,fp8,fp8,0,0.09416533509890239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,64,0,1,fp8,fp8,0,0.2030293345451355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,64,0,1,float16,fp8,0,0.21945067246754965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,64,128,1,float16,float16,0,0.0743093341588974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,64,0,1,float16,float16,0,0.21748799085617065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,64,128,1,float16,fp8,0,0.07228266696135204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,64,128,1,fp8,fp8,0,0.08772800366083781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,64,0,1,float16,fp8,0,0.21884800990422568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,64,0,1,fp8,fp8,0,0.20174932479858398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,64,128,1,float16,float16,0,0.07186133166154225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,64,128,1,float16,fp8,0,0.07249600191911061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,64,0,1,float16,float16,0,0.22349866231282553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,64,128,1,fp8,fp8,0,0.087226668993632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,64,0,1,float16,fp8,0,0.21839465697606406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,64,0,1,fp8,fp8,0,0.20240533351898193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,64,128,1,float16,float16,0,0.25150932868321735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,64,128,1,float16,fp8,0,0.24875199794769287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,64,128,1,fp8,fp8,0,0.3357066710789998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,64,0,1,float16,float16,0,1.0395253499348958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,64,128,1,float16,float16,0,0.2719893256823222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,64,0,1,fp8,fp8,0,0.9115626811981201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,64,0,1,float16,fp8,0,1.0324479738871257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,64,128,1,float16,fp8,0,0.26927467187245685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,64,128,1,fp8,fp8,0,0.35257065296173096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,64,128,1,float16,float16,0,0.15237333377202353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,64,0,1,float16,float16,0,1.0562079747517903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,64,128,1,float16,fp8,0,0.1511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,64,0,1,float16,fp8,0,1.0533920129140217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,64,0,1,fp8,fp8,0,0.9281280040740967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,64,0,1,float16,float16,0,0.5252480109532675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,64,0,1,float16,float16,0,0.5035680135091146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,64,128,1,fp8,fp8,0,0.20548266172409058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,64,0,1,float16,fp8,0,0.516325314839681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,64,128,1,float16,float16,0,0.13642133275667825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,64,0,1,fp8,fp8,0,0.4830720027287801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,64,128,1,float16,fp8,0,0.1341653366883596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,64,128,1,fp8,fp8,0,0.1846239964167277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,64,0,1,float16,fp8,0,0.5082506736119589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,64,0,1,fp8,fp8,0,0.4704960187276204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,64,128,1,float16,float16,0,0.14240533113479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,64,128,1,float16,fp8,0,0.14108266433080038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,64,0,1,float16,float16,0,0.5068213144938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,64,128,1,fp8,fp8,0,0.19300800561904907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,64,0,1,float16,fp8,0,0.5079946517944336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,64,128,1,fp8,fp8,0,0.12219199538230896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,64,0,1,fp8,fp8,0,0.47580798467000324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,64,128,1,float16,float16,0,0.08784000078837077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,64,0,1,float16,float16,0,0.27637867132822674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,64,128,1,float16,fp8,0,0.0798826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,64,128,1,float16,fp8,0,0.08946667114893596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,64,0,1,float16,fp8,0,0.273034671942393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,64,0,1,fp8,fp8,0,0.25782932837804157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,64,0,1,fp8,fp8,0,0.24848000208536783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,64,0,1,float16,fp8,0,0.27135467529296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,64,128,1,float16,float16,0,0.0817386656999588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,64,0,1,float16,float16,0,0.26955199241638184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,64,128,1,fp8,fp8,0,0.10937600334485371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,64,128,1,float16,float16,0,0.08387200037638347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,64,128,1,float16,fp8,0,0.08431466420491536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,64,128,1,fp8,fp8,0,0.11523200074831645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,64,0,1,float16,float16,0,0.27059733867645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,64,0,1,float16,fp8,0,0.27145065863927204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,64,0,1,fp8,fp8,0,0.25173334280649823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,64,128,1,float16,float16,0,0.06499733527501424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,64,0,1,float16,float16,0,0.16356799999872842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,64,128,1,float16,fp8,0,0.06393066545327504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,64,128,1,fp8,fp8,0,0.08142399787902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,64,0,1,float16,fp8,0,0.16176533699035645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,64,0,1,fp8,fp8,0,0.1511679987112681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,64,128,1,float16,float16,0,0.06342400113741557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,64,0,1,float16,float16,0,0.16265066464742026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,64,128,1,float16,fp8,0,0.06356266637643178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,64,128,1,fp8,fp8,0,0.0735999991496404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,64,0,1,float16,fp8,0,0.16338133811950684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,64,0,1,fp8,fp8,0,0.1509119967619578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,64,128,1,float16,float16,0,0.06374399860699971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,64,0,1,fp8,fp8,0,0.15134400129318237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,64,0,1,float16,float16,0,0.1629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,64,128,1,float16,fp8,0,0.06338133414586385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,64,128,1,fp8,fp8,0,0.07543999950091045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,64,0,1,float16,fp8,0,0.16210132837295532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,64,128,1,float16,float16,0,0.3927893241246541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,64,128,1,float16,fp8,0,0.3898719946543376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,64,128,1,fp8,fp8,0,0.5044266780217489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,64,0,1,float16,float16,0,1.3513120015462239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,64,0,1,float16,fp8,0,1.3458986282348633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,64,128,1,float16,float16,0,0.41913068294525146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,64,128,1,float16,fp8,0,0.4152373472849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,64,0,1,fp8,fp8,0,1.2520053386688232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,64,128,1,fp8,fp8,0,0.5301173528035482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,64,0,1,float16,float16,0,1.3796745936075847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,64,0,1,float16,fp8,0,1.3720693588256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,64,128,1,float16,float16,0,0.231930673122406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,64,128,1,float16,fp8,0,0.22965866327285767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,64,128,1,fp8,fp8,0,0.29877867301305133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,64,0,1,float16,float16,0,0.7342987060546875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,64,0,1,fp8,fp8,0,1.2723093032836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,64,128,1,float16,float16,0,0.20244799057642618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,64,0,1,float16,fp8,0,0.7237546443939209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,64,128,1,float16,fp8,0,0.19968533515930176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,64,0,1,fp8,fp8,0,0.637440005938212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,64,128,1,fp8,fp8,0,0.2714560031890869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,64,128,1,float16,float16,0,0.2195039987564087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,64,0,1,float16,float16,0,0.7055413722991943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,64,128,1,float16,fp8,0,0.21629865964253744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,64,0,1,fp8,fp8,0,0.611568013827006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,64,0,1,float16,fp8,0,0.6973919868469238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,64,128,1,fp8,fp8,0,0.2844373385111491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,64,128,1,float16,float16,0,0.12501333157221475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,64,0,1,float16,float16,0,0.715440034866333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,64,0,1,fp8,fp8,0,0.6288106838862101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,64,128,1,float16,fp8,0,0.12311999996503194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,64,0,1,float16,fp8,0,0.7146986325581869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,64,0,1,float16,float16,0,0.350655992825826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,64,128,1,fp8,fp8,0,0.16867733001708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,64,0,1,float16,fp8,0,0.3508106470108032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,64,128,1,float16,float16,0,0.11257066329320271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,64,0,1,fp8,fp8,0,0.33178667227427167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,64,128,1,float16,fp8,0,0.110944002866745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,64,0,1,float16,float16,0,0.3413013219833374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,64,0,1,float16,float16,0,0.3471946716308594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,64,128,1,fp8,fp8,0,0.15442132949829102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,64,0,1,float16,fp8,0,0.3391573429107666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,64,128,1,float16,float16,0,0.1164479951063792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,64,0,1,fp8,fp8,0,0.3187519907951355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,64,128,1,float16,fp8,0,0.11564266681671143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,64,128,1,fp8,fp8,0,0.1606559952100118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,64,0,1,float16,fp8,0,0.34219201405843097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,64,128,1,float16,float16,0,0.07522666454315186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,64,0,1,fp8,fp8,0,0.32810133695602417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,64,0,1,float16,float16,0,0.18555200099945068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,64,128,1,float16,fp8,0,0.07533866663773854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,64,128,1,fp8,fp8,0,0.10347200433413188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,64,0,1,float16,fp8,0,0.1870666742324829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,64,0,1,fp8,fp8,0,0.17813332875569662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,64,128,1,float16,float16,0,0.06875733534495036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,64,128,1,float16,fp8,0,0.06906133393446605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,64,0,1,float16,float16,0,0.18369599183400473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,64,128,1,fp8,fp8,0,0.08755733569463094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,64,0,1,float16,fp8,0,0.18185599644978842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,64,0,1,fp8,fp8,0,0.1704746683438619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,64,128,1,float16,float16,0,0.07140799860159557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,64,0,1,float16,float16,0,0.1827146609624227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,64,128,1,float16,fp8,0,0.07054933408896129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,64,128,1,fp8,fp8,0,0.09579199552536011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,64,0,1,float16,fp8,0,0.18450133005777994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,64,0,1,fp8,fp8,0,0.1732693314552307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,64,128,1,float16,float16,0,0.05460800230503082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,64,0,1,float16,float16,0,0.1081119974454244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,64,128,1,float16,fp8,0,0.05509866774082184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,64,128,1,fp8,fp8,0,0.06509866813818614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,64,128,1,float16,fp8,0,0.05486399928728739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,64,0,1,float16,fp8,0,0.1088800032933553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,64,128,1,float16,float16,0,0.0544053316116333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,64,0,1,fp8,fp8,0,0.10348266363143921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,64,0,1,float16,float16,0,0.10806933045387268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,64,128,1,fp8,fp8,0,0.06425599753856659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,64,0,1,float16,fp8,0,0.10899200042088826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,64,0,1,fp8,fp8,0,0.10236266255378723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,64,128,1,float16,float16,0,0.055061335364977516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,64,0,1,float16,float16,0,0.10813333590825398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,64,128,1,float16,fp8,0,0.05473066866397858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,64,128,1,fp8,fp8,0,0.06465066472689311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,64,0,1,float16,fp8,0,0.10811733206113179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,64,0,1,fp8,fp8,0,0.10337066650390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,64,128,1,float16,float16,0,0.29652265707651776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,64,128,1,float16,fp8,0,0.2916053334871928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,64,128,1,fp8,fp8,0,0.3858400185902913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,64,0,1,float16,float16,0,0.8238666852315267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,64,128,1,float16,float16,0,0.32062933842341107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,64,0,1,float16,fp8,0,0.8153119881947836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,64,0,1,fp8,fp8,0,0.7370506922403971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,64,128,1,float16,fp8,0,0.3168746630350749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,64,128,1,fp8,fp8,0,0.4057600100835164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,64,0,1,float16,float16,0,0.8437653382619222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,64,128,1,float16,float16,0,0.17781867583592734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,64,0,1,float16,fp8,0,0.8413919607798258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,64,0,1,fp8,fp8,0,0.7534879843393961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,64,128,1,float16,fp8,0,0.17537067333857217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,64,0,1,float16,float16,0,0.42139732837677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,64,128,1,fp8,fp8,0,0.2323733369509379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,64,0,1,float16,fp8,0,0.4189600149790446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,64,128,1,float16,float16,0,0.15576533476511636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,64,0,1,fp8,fp8,0,0.39535999298095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,64,128,1,float16,fp8,0,0.15337600310643515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,64,0,1,float16,float16,0,0.3990773359934489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,64,128,1,fp8,fp8,0,0.20907733837763467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,64,0,1,float16,fp8,0,0.39610131581624347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,64,128,1,float16,float16,0,0.1649386684099833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,64,0,1,fp8,fp8,0,0.37402133146921795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,64,0,1,float16,float16,0,0.40755200386047363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,64,128,1,float16,fp8,0,0.1622986694176992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,64,128,1,fp8,fp8,0,0.2214720050493876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,64,0,1,float16,fp8,0,0.41172266006469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,64,128,1,float16,float16,0,0.09877333045005798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,64,0,1,fp8,fp8,0,0.3860693375269572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,64,0,1,float16,float16,0,0.21708800395329794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,64,128,1,float16,fp8,0,0.09891733527183533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,64,128,1,fp8,fp8,0,0.13556800285975137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,64,0,1,float16,fp8,0,0.2185386617978414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,64,0,1,fp8,fp8,0,0.2116533319155375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,64,128,1,float16,float16,0,0.09082133571306865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,64,0,1,float16,float16,0,0.21053866545359293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,64,128,1,float16,fp8,0,0.08854933579762776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,64,128,1,fp8,fp8,0,0.12276267011960347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,64,0,1,float16,fp8,0,0.21057599782943726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,64,128,1,float16,float16,0,0.09236799677213033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,64,128,1,fp8,fp8,0,0.12853333353996277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,64,0,1,fp8,fp8,0,0.1982826590538025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,64,0,1,float16,float16,0,0.12372266252835591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,64,128,1,float16,fp8,0,0.09080533186594646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,64,128,1,float16,fp8,0,0.05983466903368632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,64,128,1,fp8,fp8,0,0.08496532837549846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,64,0,1,float16,float16,0,0.214303990205129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,64,0,1,float16,fp8,0,0.21475734313329062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,64,128,1,float16,float16,0,0.05862933397293091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,64,0,1,fp8,fp8,0,0.20229333639144897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,64,0,1,float16,fp8,0,0.12433066964149475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,64,0,1,fp8,fp8,0,0.11929600437482198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,64,128,1,float16,float16,0,0.054986665646235146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,64,0,1,float16,float16,0,0.12082133690516154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,64,128,1,float16,fp8,0,0.055439998706181846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,64,128,1,fp8,fp8,0,0.07267733414967854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,64,0,1,float16,fp8,0,0.1220906674861908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,64,0,1,fp8,fp8,0,0.1139413317044576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,64,128,1,float16,float16,0,0.05611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,64,0,1,float16,float16,0,0.12141866485277812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,64,128,1,float16,fp8,0,0.0562666654586792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,64,128,1,float16,fp8,0,0.04910400013128916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,64,128,1,fp8,fp8,0,0.07537066439787547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,64,0,1,float16,fp8,0,0.12237866719563802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,64,0,1,float16,fp8,0,0.08334400256474812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,64,0,1,fp8,fp8,0,0.08046933511892955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,64,0,1,fp8,fp8,0,0.11545600493748982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,64,128,1,float16,float16,0,0.04849066833655039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,64,0,1,float16,float16,0,0.08345066507657369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,64,128,1,fp8,fp8,0,0.050437331199645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,64,128,1,float16,float16,0,0.045226668318112694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,64,0,1,float16,float16,0,0.08281066517035167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,64,128,1,float16,fp8,0,0.04752533137798309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,64,128,1,fp8,fp8,0,0.05044800043106079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,64,0,1,float16,fp8,0,0.08375466863314311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,64,0,1,fp8,fp8,0,0.08026666442553203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,64,128,1,float16,float16,0,0.04877866804599762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,64,0,1,float16,float16,0,0.08295466502507527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,64,128,1,float16,fp8,0,0.04602666695912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,64,128,1,fp8,fp8,0,0.050442665815353394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,64,0,1,float16,fp8,0,0.083146666487058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,64,0,1,fp8,fp8,0,0.0804799993832906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,64,128,1,float16,float16,0,0.39506133397420246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,64,128,1,float16,fp8,0,0.38761067390441895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,64,0,1,float16,float16,0,0.821664015452067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,64,128,1,fp8,fp8,0,0.49958399931589764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,64,128,1,float16,float16,0,0.42338132858276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,64,0,1,float16,fp8,0,0.813098669052124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,64,0,1,fp8,fp8,0,0.7530773480733236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,64,0,1,float16,float16,0,0.8420373598734537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,64,128,1,float16,fp8,0,0.4153493245442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,64,128,1,fp8,fp8,0,0.527237335840861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,64,0,1,fp8,fp8,0,0.7797333399454752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,64,128,1,float16,float16,0,0.23372799158096313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,64,128,1,float16,fp8,0,0.22904000679651895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,64,0,1,float16,float16,0,0.4500053326288859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,64,128,1,float16,float16,0,0.2009119987487793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,64,0,1,float16,fp8,0,0.8367679913838705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,64,128,1,fp8,fp8,0,0.2946186661720276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,64,0,1,float16,fp8,0,0.4471413294474284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,64,0,1,fp8,fp8,0,0.40326933066050213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,64,0,1,float16,fp8,0,0.41656001408894855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,64,128,1,float16,fp8,0,0.19843200842539468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,64,0,1,float16,float16,0,0.4241866668065389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,64,128,1,fp8,fp8,0,0.2682933410008748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,64,0,1,fp8,fp8,0,0.37644267082214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,64,128,1,float16,float16,0,0.21982399622599283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,64,128,1,float16,fp8,0,0.21568000316619873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,64,128,1,fp8,fp8,0,0.28246933221817017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,64,0,1,float16,float16,0,0.4387893279393514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,64,128,1,float16,float16,0,0.12351466218630473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,64,0,1,float16,fp8,0,0.43558398882548016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,64,128,1,fp8,fp8,0,0.16532267133394876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,64,0,1,fp8,fp8,0,0.3908426761627197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,64,0,1,float16,float16,0,0.21931199232737222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,64,128,1,float16,fp8,0,0.1213759978612264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,64,0,1,float16,fp8,0,0.21983466545740762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,64,0,1,fp8,fp8,0,0.2105706731478373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,64,128,1,float16,float16,0,0.11117866635322571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,64,0,1,float16,float16,0,0.20782933632532755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,64,128,1,float16,fp8,0,0.10894933342933655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,64,128,1,fp8,fp8,0,0.14942399660746256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,64,0,1,float16,fp8,0,0.20507733027140299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,64,0,1,fp8,fp8,0,0.196234663327535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,64,128,1,float16,float16,0,0.11388799548149109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,64,128,1,float16,fp8,0,0.11388799548149109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,64,0,1,float16,float16,0,0.21116799116134644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,64,128,1,fp8,fp8,0,0.15730133652687073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,64,0,1,float16,fp8,0,0.20943466822306314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,64,0,1,fp8,fp8,0,0.20213866233825684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,64,128,1,float16,float16,0,0.07193600138028462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,64,0,1,float16,float16,0,0.11678399642308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,64,128,1,float16,fp8,0,0.07188799977302551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,64,128,1,fp8,fp8,0,0.10051733255386353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,64,0,1,float16,fp8,0,0.11942399541536967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,64,0,1,fp8,fp8,0,0.11412800351778667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,64,128,1,float16,float16,0,0.0642133355140686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,64,0,1,float16,float16,0,0.1127786636352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,64,128,1,float16,fp8,0,0.06390400230884552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,64,128,1,fp8,fp8,0,0.08281066517035167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,64,0,1,float16,fp8,0,0.11117333173751831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,64,0,1,fp8,fp8,0,0.10624000430107117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,64,128,1,float16,float16,0,0.06614933411280315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,64,0,1,float16,float16,0,0.11413333813349406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,64,128,1,float16,fp8,0,0.06808533271153767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,64,128,1,fp8,fp8,0,0.09055999914805095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,64,128,1,float16,fp8,0,0.045978665351867676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,64,0,1,float16,fp8,0,0.1146613359451294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,64,0,1,fp8,fp8,0,0.1083679993947347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,64,128,1,float16,float16,0,0.045040001471837364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,64,0,1,float16,float16,0,0.06650133430957794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,64,128,1,fp8,fp8,0,0.0631039987007777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,64,0,1,float16,fp8,0,0.0676693320274353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,64,0,1,fp8,fp8,0,0.06524799764156342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,64,128,1,float16,float16,0,0.044666667779286705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,64,0,1,float16,float16,0,0.06596800188223521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,64,128,1,float16,fp8,0,0.044591998060544334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,64,128,1,fp8,fp8,0,0.05525333185990652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,64,128,1,fp8,fp8,0,0.055829331278800964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,64,0,1,float16,fp8,0,0.0662613312403361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,64,0,1,float16,fp8,0,0.06677866478761037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,64,0,1,fp8,fp8,0,0.06322133541107178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,64,128,1,float16,float16,0,0.04457066456476847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,64,0,1,float16,float16,0,0.06619733572006226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,64,128,1,float16,fp8,0,0.0452106644709905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,64,0,1,fp8,fp8,0,0.06377066671848297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,64,128,1,float16,float16,0,0.03379199902216593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,64,0,1,float16,float16,0,0.06083733340104421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,64,128,1,float16,fp8,0,0.03443733354409536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,64,128,1,fp8,fp8,0,0.04062400013208389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,64,0,1,float16,fp8,0,0.06069866816202799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,64,0,1,fp8,fp8,0,0.05936000247796377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,64,128,1,float16,float16,0,0.03422400106986364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,64,0,1,float16,float16,0,0.06155733267466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,64,128,1,float16,fp8,0,0.03392533212900162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,64,128,1,fp8,fp8,0,0.04027733455101649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,64,0,1,float16,fp8,0,0.06145599981149038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,64,0,1,fp8,fp8,0,0.059301331639289856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,64,128,1,float16,float16,0,0.033930666744709015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,64,0,1,float16,float16,0,0.06057600180308024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,64,128,1,float16,fp8,0,0.03440533330043157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,64,128,1,fp8,fp8,0,0.0403413325548172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,64,0,1,float16,fp8,0,0.0609493354956309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,64,0,1,fp8,fp8,0,0.05932799975077311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,64,128,1,float16,float16,0,0.2977546652158101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,64,128,1,float16,float16,0,0.3129173318545024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,64,128,1,float16,fp8,0,0.2913386623064677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,64,0,1,float16,float16,0,0.5218666791915894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,64,128,1,fp8,fp8,0,0.37570667266845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,64,0,1,float16,fp8,0,0.507365345954895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,64,0,1,fp8,fp8,0,0.46026134490966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,64,128,1,float16,fp8,0,0.307584007581075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,64,0,1,float16,float16,0,0.5297866662343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,64,128,1,fp8,fp8,0,0.39349865913391113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,64,128,1,float16,float16,0,0.17670400937398276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,64,0,1,float16,float16,0,0.27780266602834064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,64,128,1,float16,fp8,0,0.1742453376452128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,64,0,1,fp8,fp8,0,0.47566401958465576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,64,0,1,float16,fp8,0,0.5258986552556356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,64,128,1,fp8,fp8,0,0.227183997631073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,64,0,1,float16,fp8,0,0.2742026646931966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,64,0,1,fp8,fp8,0,0.25941866636276245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,64,128,1,float16,float16,0,0.15530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,64,0,1,float16,float16,0,0.25135467449824017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,64,128,1,float16,fp8,0,0.15371732910474142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,64,128,1,fp8,fp8,0,0.20875734090805054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,64,0,1,float16,fp8,0,0.25042132536570233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,64,0,1,float16,float16,0,0.25995733340581256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,64,128,1,float16,fp8,0,0.16114133596420288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,64,0,1,fp8,fp8,0,0.24025066693623862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,64,128,1,float16,float16,0,0.16397333145141602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,64,128,1,fp8,fp8,0,0.21638399362564087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,64,0,1,float16,fp8,0,0.2572213411331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,64,128,1,float16,float16,0,0.09667199850082397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,64,0,1,float16,float16,0,0.14289599657058716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,64,0,1,fp8,fp8,0,0.2491040031115214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,64,128,1,float16,fp8,0,0.09590933720270793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,64,128,1,fp8,fp8,0,0.1320746640364329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,64,0,1,float16,fp8,0,0.14432000120480856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,64,0,1,fp8,fp8,0,0.14037866393725076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,64,128,1,float16,float16,0,0.08694400389989217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,64,0,1,float16,float16,0,0.13488533099492392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,64,128,1,float16,fp8,0,0.08637866377830505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,64,128,1,fp8,fp8,0,0.12110933661460876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,64,0,1,float16,fp8,0,0.13409066200256348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,64,0,1,fp8,fp8,0,0.13025066256523132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,64,128,1,float16,float16,0,0.09065066774686177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,64,0,1,float16,float16,0,0.13693333665529886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,64,128,1,float16,fp8,0,0.09055466453234355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,64,128,1,fp8,fp8,0,0.12502400080362955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,64,0,1,float16,fp8,0,0.1363040010134379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,64,0,1,fp8,fp8,0,0.13318399588267008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,64,128,1,float16,float16,0,0.057376002271970115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,64,0,1,float16,float16,0,0.08258133133252461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,64,128,1,float16,fp8,0,0.05670933425426483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,64,128,1,fp8,fp8,0,0.08064533273379008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,64,0,1,float16,fp8,0,0.08146133522192638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,64,0,1,fp8,fp8,0,0.0783733328183492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,64,0,1,fp8,fp8,0,0.07384000221888225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,64,128,1,float16,float16,0,0.05175999800364176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,64,0,1,float16,float16,0,0.07774933179219563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,64,128,1,float16,fp8,0,0.051957334081331887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,64,128,1,fp8,fp8,0,0.07160533467928569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,64,128,1,fp8,fp8,0,0.07014399766921997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,64,0,1,float16,fp8,0,0.07740800082683563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,64,128,1,float16,float16,0,0.0524586687485377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,64,0,1,float16,float16,0,0.07712533573309581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,64,128,1,float16,fp8,0,0.054431999723116554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,64,0,1,float16,fp8,0,0.07834133505821228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,64,0,1,fp8,fp8,0,0.07493333518505096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,64,128,1,float16,float16,0,0.0415040006240209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,64,0,1,float16,float16,0,0.050154666105906166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,64,128,1,float16,fp8,0,0.04186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,64,128,1,fp8,fp8,0,0.04426133135954539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,64,0,1,float16,fp8,0,0.05083199838797251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,64,0,1,fp8,fp8,0,0.05060266455014547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,64,128,1,float16,float16,0,0.038218667109807335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,64,0,1,float16,float16,0,0.050698667764663696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,64,128,1,float16,fp8,0,0.04139200101296107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,64,128,1,fp8,fp8,0,0.043162668744723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,64,0,1,float16,fp8,0,0.050554667909940086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,64,0,1,fp8,fp8,0,0.049039999643961586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,64,128,1,float16,float16,0,0.041152000427246094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,64,0,1,float16,float16,0,0.050517335534095764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,64,128,1,float16,fp8,0,0.03900266687075297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,64,128,1,fp8,fp8,0,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,64,0,1,float16,fp8,0,0.05060266455014547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,64,0,1,fp8,fp8,0,0.049551998575528465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,64,128,1,float16,float16,0,0.030970667799313862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,64,0,1,float16,float16,0,0.0495413343111674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,64,0,1,float16,float16,0,0.049125333627065025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,64,128,1,float16,fp8,0,0.030960001051425934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,64,128,1,fp8,fp8,0,0.037717332442601524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,64,0,1,float16,fp8,0,0.05008000135421753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,64,0,1,fp8,fp8,0,0.04889066517353058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,64,128,1,float16,float16,0,0.031109333038330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,64,128,1,float16,fp8,0,0.03125333289305369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,64,128,1,fp8,fp8,0,0.03736533224582672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,64,128,1,fp8,fp8,0,0.037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,64,0,1,float16,fp8,0,0.04959466556708018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,64,0,1,fp8,fp8,0,0.04897599915663401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,64,128,1,float16,float16,0,0.031130666534105938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,64,0,1,float16,float16,0,0.0498933345079422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,64,128,1,float16,fp8,0,0.031114667654037476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,64,0,1,float16,fp8,0,0.0499893327554067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,64,0,1,fp8,fp8,0,0.048613334695498146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,64,128,1,float16,float16,0,0.3774720033009847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,64,0,1,float16,float16,0,0.535210649172465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,64,128,1,float16,fp8,0,0.3722720146179199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,64,128,1,fp8,fp8,0,0.47791465123494464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,64,0,1,float16,fp8,0,0.5323573350906372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,64,0,1,fp8,fp8,0,0.4933866659800212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,64,128,1,float16,float16,0,0.39447466532389325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,64,0,1,float16,float16,0,0.5469173192977905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,64,0,1,float16,float16,0,0.3068693280220032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,64,128,1,fp8,fp8,0,0.4980693260828654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,64,128,1,float16,fp8,0,0.38791465759277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,64,0,1,float16,fp8,0,0.5408106644948324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,64,128,1,float16,float16,0,0.2262986699740092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,64,0,1,fp8,fp8,0,0.5108693440755209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,64,128,1,float16,fp8,0,0.2216320037841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,64,128,1,fp8,fp8,0,0.2832746704419454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,64,0,1,float16,fp8,0,0.3021226723988851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,64,0,1,fp8,fp8,0,0.2767519950866699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,64,128,1,float16,float16,0,0.1990293264389038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,64,0,1,float16,float16,0,0.27897600332895917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,64,128,1,float16,fp8,0,0.1946880022684733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,64,128,1,fp8,fp8,0,0.259061336517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,64,0,1,float16,fp8,0,0.27614933252334595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,64,0,1,fp8,fp8,0,0.25177599986394245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,64,128,1,float16,float16,0,0.20644267400105795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,64,0,1,float16,float16,0,0.2861013412475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,64,0,1,float16,fp8,0,0.28564266363779706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,64,128,1,float16,fp8,0,0.20489599307378134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,64,0,1,float16,float16,0,0.15493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,64,128,1,fp8,fp8,0,0.27002666393915814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,64,128,1,float16,float16,0,0.12426666418711345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,64,0,1,fp8,fp8,0,0.26155734062194824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,64,128,1,float16,fp8,0,0.1209279994169871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,64,128,1,fp8,fp8,0,0.16289599736531576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,64,0,1,float16,fp8,0,0.15398400028546652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,64,0,1,fp8,fp8,0,0.14827199776967367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,64,128,1,float16,float16,0,0.11041067043940227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,64,0,1,float16,float16,0,0.1404213309288025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,64,128,1,float16,fp8,0,0.10790933171908061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,64,128,1,fp8,fp8,0,0.14909866452217102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,64,0,1,float16,fp8,0,0.13937600453694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,64,0,1,fp8,fp8,0,0.1357439955075582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,64,128,1,float16,float16,0,0.11162666479746501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,64,0,1,float16,float16,0,0.1434293289979299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,64,128,1,float16,fp8,0,0.11226133505503337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,64,128,1,fp8,fp8,0,0.15411733587582907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,64,0,1,float16,fp8,0,0.14362667004267374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,64,0,1,fp8,fp8,0,0.14198933045069376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,64,128,1,float16,float16,0,0.06916800141334534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,64,0,1,float16,float16,0,0.08284266789754231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,64,128,1,float16,fp8,0,0.070933332045873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,64,128,1,fp8,fp8,0,0.09900800387064616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,64,0,1,float16,fp8,0,0.0842186709245046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,64,0,1,fp8,fp8,0,0.08457600076993306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,64,128,1,float16,float16,0,0.06392000118891399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,64,0,1,float16,float16,0,0.07794666786988576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,64,128,1,float16,fp8,0,0.06283733248710632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,64,128,1,fp8,fp8,0,0.08125333487987518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,64,0,1,float16,fp8,0,0.07745600243409474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,64,0,1,fp8,fp8,0,0.07588799794514973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,64,128,1,float16,float16,0,0.0666720022757848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,64,0,1,float16,float16,0,0.079434668024381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,64,128,1,float16,fp8,0,0.06515733400980632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,64,128,1,float16,fp8,0,0.04397333165009817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,64,128,1,fp8,fp8,0,0.08679999907811482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,64,0,1,float16,fp8,0,0.08063999811808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,64,0,1,fp8,fp8,0,0.0788373351097107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,64,128,1,float16,float16,0,0.04387733340263367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,64,0,1,float16,float16,0,0.049125333627065025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,64,128,1,fp8,fp8,0,0.05529066423575083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,64,0,1,float16,fp8,0,0.050016000866889954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,64,0,1,fp8,fp8,0,0.04986133178075155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,64,128,1,float16,float16,0,0.04228266576925913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,64,0,1,float16,float16,0,0.047775998711586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,64,128,1,float16,fp8,0,0.041984001795450844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,64,128,1,float16,fp8,0,0.04302933315436045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,64,128,1,fp8,fp8,0,0.05364799996217092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,64,0,1,float16,fp8,0,0.04716266691684723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,64,0,1,fp8,fp8,0,0.04601066807905833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,64,128,1,float16,float16,0,0.04297600189844767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,64,0,1,float16,float16,0,0.047925333182017006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,64,128,1,float16,fp8,0,0.029696000119050343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,64,128,1,fp8,fp8,0,0.053818667928377785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,64,0,1,float16,fp8,0,0.04849066833655039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,64,0,1,fp8,fp8,0,0.046426668763160706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,64,128,1,float16,float16,0,0.030320001145203907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,64,0,1,float16,float16,0,0.03919466584920883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,64,128,1,fp8,fp8,0,0.03676799933115641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,64,0,1,float16,fp8,0,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,64,0,1,fp8,fp8,0,0.03884266565243403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,64,128,1,float16,float16,0,0.02979733298222224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,64,0,1,float16,float16,0,0.03941866755485535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,64,128,1,float16,fp8,0,0.02942933390537898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,64,128,1,fp8,fp8,0,0.03637866675853729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,64,0,1,float16,fp8,0,0.039349332451820374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,64,0,1,fp8,fp8,0,0.03895466774702072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,64,128,1,float16,float16,0,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,64,0,1,float16,float16,0,0.0390133336186409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,64,128,1,float16,fp8,0,0.030229332546393078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,64,128,1,fp8,fp8,0,0.036271999279658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,64,0,1,float16,fp8,0,0.03967999915281931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,64,0,1,fp8,fp8,0,0.039434666434923805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,64,128,1,float16,float16,0,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,64,0,1,float16,float16,0,0.03868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,64,128,1,float16,fp8,0,0.028058665494124096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,64,128,1,fp8,fp8,0,0.03438399980465571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,64,0,1,float16,fp8,0,0.03922666609287262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,64,0,1,fp8,fp8,0,0.03777066618204117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,64,128,1,float16,float16,0,0.027888000011444092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,64,0,1,float16,float16,0,0.039077334105968475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,64,128,1,float16,fp8,0,0.027823999524116516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,64,128,1,fp8,fp8,0,0.03469866762558619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,64,0,1,float16,fp8,0,0.03842666745185852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,64,0,1,fp8,fp8,0,0.03736000011364619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,64,0,1,fp8,fp8,0,0.03719466676314672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,64,128,1,float16,float16,0,0.027957332630952198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,64,0,1,float16,float16,0,0.0383093332250913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,64,128,1,float16,fp8,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,64,128,1,fp8,fp8,0,0.03521066655715307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,64,0,1,float16,fp8,0,0.03867733230193456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,64,128,1,float16,float16,0,0.3133546710014343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,64,0,1,float16,fp8,0,0.3795573314030965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,64,0,1,float16,float16,0,0.3845813274383545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,64,128,1,float16,fp8,0,0.3048373262087504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,64,128,1,fp8,fp8,0,0.38871467113494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,64,0,1,fp8,fp8,0,0.34956268469492596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,64,128,1,float16,float16,0,0.32465600967407227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,64,0,1,float16,float16,0,0.39662933349609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,64,128,1,float16,fp8,0,0.31649599472681683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,64,128,1,fp8,fp8,0,0.39658133188883465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,64,0,1,float16,fp8,0,0.39081064860026044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,64,128,1,float16,float16,0,0.1846346656481425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,64,0,1,fp8,fp8,0,0.35499731699625653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,64,0,1,float16,float16,0,0.23215466737747192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,64,128,1,float16,fp8,0,0.18039999405543009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,64,128,1,fp8,fp8,0,0.22901866833368936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,64,0,1,float16,fp8,0,0.2266719937324524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,64,128,1,fp8,fp8,0,0.21432000398635864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,64,0,1,fp8,fp8,0,0.1992959976196289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,64,128,1,float16,float16,0,0.16390933593114218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,64,0,1,fp8,fp8,0,0.18290134270985922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,64,0,1,float16,float16,0,0.1869600017865499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,64,128,1,float16,fp8,0,0.16019200285275778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,64,0,1,float16,fp8,0,0.1820746660232544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,64,128,1,float16,float16,0,0.17308266957600912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,64,0,1,float16,float16,0,0.19912532965342203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,64,128,1,float16,fp8,0,0.16804800430933634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,64,128,1,fp8,fp8,0,0.21818667650222778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,64,0,1,float16,fp8,0,0.19181867440541586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,64,0,1,fp8,fp8,0,0.18714133898417154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,64,128,1,float16,float16,0,0.10051733255386353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,64,0,1,float16,float16,0,0.10846400260925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,64,128,1,float16,fp8,0,0.09895466764767964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,64,128,1,fp8,fp8,0,0.1346399982770284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,64,0,1,float16,fp8,0,0.1069493293762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,64,0,1,fp8,fp8,0,0.1109279990196228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,64,128,1,float16,float16,0,0.08747733632723491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,64,0,1,float16,float16,0,0.09804800152778625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,64,128,1,float16,fp8,0,0.08524800340334575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,64,128,1,fp8,fp8,0,0.11559466520945232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,64,0,1,float16,fp8,0,0.09820266564687093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,64,0,1,fp8,fp8,0,0.10019733508427937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,64,128,1,float16,float16,0,0.09155199925104777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,64,0,1,float16,float16,0,0.10099732875823975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,64,128,1,float16,fp8,0,0.09119466940561931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,64,128,1,fp8,fp8,0,0.12494933605194092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,64,0,1,float16,fp8,0,0.10076266527175903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,64,0,1,fp8,fp8,0,0.10291733344395955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,64,128,1,float16,float16,0,0.0558186670144399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,64,128,1,float16,float16,0,0.05179200073083242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,64,0,1,float16,float16,0,0.06126399834950765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,64,128,1,float16,fp8,0,0.05418133238951365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,64,128,1,fp8,fp8,0,0.07824533184369405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,64,0,1,float16,fp8,0,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,64,0,1,fp8,fp8,0,0.06193066636721293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,64,0,1,float16,float16,0,0.05783999959627787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,64,128,1,float16,fp8,0,0.0518506666024526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,64,128,1,fp8,fp8,0,0.06828799843788147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,64,0,1,float16,fp8,0,0.05810666580994924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,64,0,1,fp8,fp8,0,0.05604266623655955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,64,128,1,float16,float16,0,0.05297600229581197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,64,0,1,float16,float16,0,0.0587360014518102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,64,128,1,float16,fp8,0,0.05275199810663859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,64,128,1,float16,fp8,0,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,64,128,1,fp8,fp8,0,0.07041599849859874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,64,0,1,float16,fp8,0,0.05919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,64,0,1,fp8,fp8,0,0.05766933163007101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,64,128,1,float16,float16,0,0.039135999977588654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,64,0,1,float16,float16,0,0.03871466716130575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,64,128,1,fp8,fp8,0,0.04365866879622141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,64,0,1,float16,fp8,0,0.03938666731119156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,64,0,1,fp8,fp8,0,0.03758399933576584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,64,128,1,float16,float16,0,0.03996799886226654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,64,0,1,float16,float16,0,0.03806400050719579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,64,128,1,float16,fp8,0,0.03624533365170161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,64,128,1,fp8,fp8,0,0.04284800092379252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,64,0,1,float16,fp8,0,0.03756266583998998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,64,0,1,fp8,fp8,0,0.035317334036032356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,64,128,1,float16,float16,0,0.04030400017897288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,64,0,1,float16,float16,0,0.037946666280428566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,64,128,1,float16,fp8,0,0.04037333279848099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,64,128,1,fp8,fp8,0,0.042693331837654114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,64,0,1,float16,fp8,0,0.03829866647720337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,64,0,1,fp8,fp8,0,0.03723733375469843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,64,128,1,float16,float16,0,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,64,0,1,float16,float16,0,0.033770665526390076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,64,128,1,float16,fp8,0,0.027877333263556164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,64,128,1,fp8,fp8,0,0.03453333427508672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,64,0,1,float16,fp8,0,0.03425599883000056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,64,0,1,fp8,fp8,0,0.03324799984693527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,64,128,1,float16,float16,0,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,64,0,1,float16,float16,0,0.033999999364217125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,64,128,1,float16,fp8,0,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,64,128,1,fp8,fp8,0,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,64,0,1,float16,fp8,0,0.03345600018898646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,64,0,1,fp8,fp8,0,0.03289599965016047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,64,128,1,float16,float16,0,0.02788266787926356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,64,0,1,float16,float16,0,0.034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,64,128,1,float16,fp8,0,0.027834666272004444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,64,128,1,fp8,fp8,0,0.03482666611671448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,64,0,1,float16,fp8,0,0.03403733422358831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,64,128,1,fp8,fp8,0,0.0329120010137558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,64,0,1,fp8,fp8,0,0.03268799930810928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,64,128,1,float16,float16,0,0.026394667724768322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,64,128,1,float16,float16,0,0.026608000199000042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,64,0,1,float16,float16,0,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,64,128,1,float16,fp8,0,0.027280000348885853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,64,0,1,float16,fp8,0,0.03330666571855545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,64,0,1,fp8,fp8,0,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,64,0,1,float16,float16,0,0.032730666299661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,64,128,1,float16,fp8,0,0.02659733345111211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,64,128,1,fp8,fp8,0,0.033173332611719765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,64,0,1,float16,fp8,0,0.03331733246644338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,64,128,1,fp8,fp8,0,0.03347733368476232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,64,0,1,fp8,fp8,0,0.03232000023126602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,64,128,1,float16,float16,0,0.026842666169007618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,64,0,1,float16,float16,0,0.03290133426586787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,64,128,1,float16,fp8,0,0.02640533447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,64,0,1,float16,fp8,0,0.03309866786003113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,64,0,1,fp8,fp8,0,0.03202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,64,128,1,float16,float16,0,0.3300960063934326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,64,0,1,float16,float16,0,0.3694026470184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,64,128,1,float16,fp8,0,0.3288000027338664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,64,128,1,fp8,fp8,0,0.4347199996312459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,64,0,1,float16,fp8,0,0.367520014444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,64,0,1,fp8,fp8,0,0.33905601501464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,64,128,1,float16,float16,0,0.3420106569925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,64,0,1,float16,float16,0,0.3837759892145793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,64,128,1,float16,fp8,0,0.3395040035247803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,64,128,1,fp8,fp8,0,0.4456533193588257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,64,128,1,float16,float16,0,0.20154666900634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,64,0,1,float16,fp8,0,0.37730133533477783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,64,0,1,fp8,fp8,0,0.34692267576853436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,64,0,1,float16,float16,0,0.22225600481033325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,64,128,1,float16,fp8,0,0.19898132483164468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,64,0,1,fp8,fp8,0,0.19269333283106485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,64,128,1,fp8,fp8,0,0.252128005027771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,64,0,1,float16,fp8,0,0.2182826598485311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,64,128,1,float16,float16,0,0.1777706742286682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,64,0,1,float16,float16,0,0.19922133286794028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,64,128,1,float16,fp8,0,0.17801600694656372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,64,128,1,fp8,fp8,0,0.23921600977579752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,64,0,1,float16,fp8,0,0.19850667317708334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,64,0,1,fp8,fp8,0,0.17995200554529825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,64,128,1,float16,float16,0,0.18518400192260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,64,0,1,float16,float16,0,0.20454933245976767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,64,128,1,float16,fp8,0,0.18356800079345703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,64,128,1,fp8,fp8,0,0.24272000789642334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,64,0,1,float16,fp8,0,0.20517865816752115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,64,0,1,fp8,fp8,0,0.18283732732137045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,64,128,1,float16,float16,0,0.11275200049082439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,64,0,1,float16,float16,0,0.11967999736467998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,64,128,1,float16,fp8,0,0.10974400242169698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,64,128,1,fp8,fp8,0,0.14528533816337585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,64,128,1,fp8,fp8,0,0.135861337184906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,64,0,1,float16,fp8,0,0.11620799700419109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,64,0,1,fp8,fp8,0,0.10585066676139832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,64,128,1,float16,float16,0,0.09911466638247173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,64,0,1,float16,float16,0,0.10439999898274739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,64,128,1,float16,fp8,0,0.09994133313496907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,64,0,1,float16,fp8,0,0.10525866349538167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,64,0,1,fp8,fp8,0,0.10026666522026062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,64,128,1,float16,float16,0,0.10417600472768147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,64,0,1,float16,float16,0,0.10830932855606079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,64,128,1,float16,fp8,0,0.10341333349545796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,64,128,1,fp8,fp8,0,0.14041067163149515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,64,0,1,float16,fp8,0,0.10559466481208801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,64,0,1,fp8,fp8,0,0.10162666440010071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,64,128,1,float16,float16,0,0.06417066852251689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,64,0,1,float16,float16,0,0.06470400094985962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,64,128,1,float16,fp8,0,0.063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,64,128,1,fp8,fp8,0,0.08821866909662883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,64,0,1,float16,fp8,0,0.06445333361625671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,64,0,1,fp8,fp8,0,0.06243733565012614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,64,128,1,float16,float16,0,0.05823466678460439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,64,0,1,float16,float16,0,0.0606826643149058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,64,128,1,float16,fp8,0,0.05890666445096334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,64,128,1,fp8,fp8,0,0.07678933441638947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,64,0,1,float16,fp8,0,0.05950400233268738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,64,0,1,fp8,fp8,0,0.055973331133524575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,64,128,1,float16,float16,0,0.060693333546320595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,64,0,1,float16,float16,0,0.060736000537872314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,64,128,1,float16,fp8,0,0.06067200005054474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,64,128,1,fp8,fp8,0,0.08345599969228108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,64,0,1,float16,fp8,0,0.06140799820423126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,64,0,1,fp8,fp8,0,0.05835199852784475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,64,128,1,float16,float16,0,0.04185600082079569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,64,0,1,float16,float16,0,0.040394666294256844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,64,128,1,float16,fp8,0,0.041375999649365745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,64,128,1,fp8,fp8,0,0.05082133412361145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,64,0,1,float16,fp8,0,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,64,0,1,fp8,fp8,0,0.03653866549332937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,64,128,1,float16,float16,0,0.03923733284076055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,64,0,1,float16,float16,0,0.03742400060097376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,64,128,1,float16,fp8,0,0.03984000037113825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,64,128,1,fp8,fp8,0,0.05120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,64,0,1,float16,fp8,0,0.03793066740036011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,64,128,1,fp8,fp8,0,0.052005335688591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,64,0,1,fp8,fp8,0,0.035002666215101876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,64,128,1,float16,float16,0,0.03988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,64,0,1,float16,float16,0,0.038890667259693146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,64,128,1,float16,fp8,0,0.03908266623814901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,64,0,1,float16,fp8,0,0.03859733293453852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,64,0,1,fp8,fp8,0,0.03602133442958196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,64,128,1,float16,float16,0,0.027653334041436512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,64,0,1,float16,float16,0,0.029253333806991577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,64,128,1,float16,fp8,0,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,64,128,1,fp8,fp8,0,0.0345920001467069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,64,128,1,float16,fp8,0,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,64,0,1,float16,fp8,0,0.02938666691382726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,64,0,1,fp8,fp8,0,0.02834133307139079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,64,128,1,float16,float16,0,0.02610666553179423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,64,0,1,float16,float16,0,0.028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,64,128,1,float16,fp8,0,0.026975999275843304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,64,128,1,fp8,fp8,0,0.03284800052642822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,64,0,1,float16,fp8,0,0.029029332101345062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,64,0,1,fp8,fp8,0,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,64,128,1,float16,float16,0,0.02638400097688039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,64,0,1,float16,float16,0,0.02890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,64,128,1,fp8,fp8,0,0.03334933271010717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,64,0,1,float16,fp8,0,0.02908266584078471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,64,0,1,fp8,fp8,0,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,64,128,1,float16,float16,0,0.024282666544119518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,64,0,1,float16,float16,0,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,64,128,1,float16,fp8,0,0.024357333779335022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,64,128,1,fp8,fp8,0,0.03169599920511246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,64,0,1,float16,fp8,0,0.027653334041436512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,64,0,1,fp8,fp8,0,0.026202666262785595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,64,0,1,float16,fp8,0,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,64,128,1,float16,float16,0,0.023498666783173878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,64,0,1,float16,float16,0,0.026965332527955372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,64,128,1,float16,fp8,0,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,64,128,1,fp8,fp8,0,0.030671998858451843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,64,0,1,fp8,fp8,0,0.02585600068171819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,64,128,1,float16,float16,0,0.023898666103680927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,64,0,1,float16,float16,0,0.026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,64,128,1,float16,fp8,0,0.024458666642506916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,64,128,1,fp8,fp8,0,0.031386665999889374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,64,0,1,float16,fp8,0,0.027306665976842243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,64,0,1,fp8,fp8,0,0.026015999416510265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,64,128,1,float16,float16,0,0.024106666445732117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,64,0,1,fp8,fp8,0,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,64,0,1,float16,float16,0,0.025962665677070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,64,128,1,float16,fp8,0,0.02380266785621643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,64,128,1,fp8,fp8,0,0.029850666721661884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,64,0,1,float16,fp8,0,0.026613332331180573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,64,128,1,float16,float16,0,0.02365333338578542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,64,0,1,float16,float16,0,0.02625600000222524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,64,128,1,float16,fp8,0,0.024010665714740753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,64,128,1,fp8,fp8,0,0.030261332790056866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,64,0,1,float16,fp8,0,0.026133333643277485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,64,0,1,fp8,fp8,0,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,64,0,1,fp8,fp8,0,0.024671999116738636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,64,128,1,float16,float16,0,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,64,0,1,float16,float16,0,0.02619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,64,128,1,float16,fp8,0,0.024005333582560223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,64,128,1,fp8,fp8,0,0.0308746670683225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,64,0,1,float16,fp8,0,0.026416001220544178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,64,128,1,float16,float16,0,0.31704533100128174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,64,128,1,float16,fp8,0,0.31481067339579266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,64,0,1,float16,float16,0,0.30668799082438153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,64,128,1,fp8,fp8,0,0.4111040035883586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,64,0,1,float16,fp8,0,0.30590933561325073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,64,0,1,fp8,fp8,0,0.27729066212972003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,64,128,1,float16,float16,0,0.32811200618743896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,64,0,1,float16,float16,0,0.318448007106781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,64,128,1,float16,fp8,0,0.3228800098101298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,64,128,1,fp8,fp8,0,0.4198933442433675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,64,128,1,float16,float16,0,0.19499733050664267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,64,0,1,float16,fp8,0,0.3160586754480998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,64,0,1,fp8,fp8,0,0.2857759992281596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,64,0,1,float16,float16,0,0.1869386633237203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,64,128,1,float16,fp8,0,0.19033600886662802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,64,128,1,fp8,fp8,0,0.23953600724538168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,64,0,1,float16,fp8,0,0.1837493379910787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,64,0,1,fp8,fp8,0,0.16168000300725302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,64,128,1,float16,float16,0,0.17114132642745972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,64,0,1,float16,float16,0,0.1650826632976532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,64,128,1,float16,fp8,0,0.1699946721394857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,64,128,1,fp8,fp8,0,0.22593599557876587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,64,0,1,float16,fp8,0,0.16302399833997092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,64,0,1,fp8,fp8,0,0.14890666802724203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,64,128,1,float16,float16,0,0.1785386602083842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,64,128,1,float16,float16,0,0.10910933216412862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,64,0,1,float16,float16,0,0.17045332988103232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,64,128,1,float16,fp8,0,0.17715734243392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,64,128,1,fp8,fp8,0,0.22933866580327353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,64,0,1,float16,fp8,0,0.16966400543848673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,64,0,1,fp8,fp8,0,0.153957337141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,64,0,1,float16,float16,0,0.10732266306877136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,64,128,1,float16,fp8,0,0.10784533619880676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,64,0,1,float16,float16,0,0.08902399738629659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,64,128,1,fp8,fp8,0,0.13703999916712442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,64,0,1,float16,fp8,0,0.10384533802668254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,64,0,1,fp8,fp8,0,0.09276266892751057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,64,128,1,float16,float16,0,0.09548266728719075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,64,128,1,float16,fp8,0,0.09507200121879578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,64,128,1,fp8,fp8,0,0.12945066889127096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,64,0,1,float16,fp8,0,0.09005866448084514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,64,0,1,fp8,fp8,0,0.08246933420499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,64,128,1,float16,float16,0,0.10038933157920837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,64,0,1,float16,float16,0,0.09378133217493693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,64,128,1,float16,fp8,0,0.09975999593734741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,64,128,1,fp8,fp8,0,0.1307146648565928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,64,0,1,float16,fp8,0,0.09310932954152425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,64,0,1,fp8,fp8,0,0.08662399649620056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,64,128,1,float16,float16,0,0.06208533545335134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,64,0,1,float16,fp8,0,0.05693333347638448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,64,0,1,float16,float16,0,0.05754133562246958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,64,128,1,float16,fp8,0,0.062133332093556724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,64,128,1,fp8,fp8,0,0.08691199620564778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,64,0,1,fp8,fp8,0,0.053904001911481224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,64,128,1,float16,float16,0,0.055439998706181846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,64,0,1,float16,float16,0,0.05217066903909048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,64,128,1,float16,fp8,0,0.05605866511662801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,64,128,1,fp8,fp8,0,0.07299200197060902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,64,128,1,float16,fp8,0,0.05797866483529409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,64,0,1,float16,fp8,0,0.05253333350022634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,64,0,1,fp8,fp8,0,0.047600001096725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,64,128,1,float16,float16,0,0.05761066575845083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,64,0,1,float16,float16,0,0.052906667192777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,64,128,1,fp8,fp8,0,0.07849066456158955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,64,0,1,float16,fp8,0,0.05493866900602976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,64,0,1,fp8,fp8,0,0.04901333153247833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,64,0,1,float16,fp8,0,0.03535466641187668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,64,128,1,float16,float16,0,0.04073066761096319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,64,0,1,float16,float16,0,0.03649600098530451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,64,0,1,float16,float16,0,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,64,128,1,float16,fp8,0,0.04134399940570196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,64,128,1,fp8,fp8,0,0.049695998430252075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,64,128,1,fp8,fp8,0,0.05064000189304352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,64,0,1,fp8,fp8,0,0.032511999209721885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,64,128,1,float16,float16,0,0.03866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,64,128,1,float16,fp8,0,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,64,0,1,float16,fp8,0,0.03382399926582972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,64,0,1,fp8,fp8,0,0.030618667602539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,64,128,1,float16,float16,0,0.039781334499518074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,64,0,1,float16,float16,0,0.033914667864640556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,64,128,1,float16,fp8,0,0.04038933416207632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,64,128,1,fp8,fp8,0,0.05072533090909322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,64,0,1,float16,fp8,0,0.03498133271932602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,64,128,1,fp8,fp8,0,0.034287999073664345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,64,0,1,fp8,fp8,0,0.03155199935038885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,64,128,1,float16,float16,0,0.027130665878454845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,64,0,1,float16,float16,0,0.025146665672461193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,64,128,1,float16,fp8,0,0.02784000088771184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,64,0,1,float16,fp8,0,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,64,0,1,fp8,fp8,0,0.02426133304834366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,64,128,1,float16,float16,0,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,64,0,1,float16,float16,0,0.023973333338896435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,64,128,1,float16,fp8,0,0.02626666675011317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,64,128,1,fp8,fp8,0,0.033285332222779594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,64,0,1,float16,fp8,0,0.024501333634058636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,64,0,1,fp8,fp8,0,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,64,0,1,float16,fp8,0,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,64,128,1,float16,float16,0,0.02701333413521449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,64,0,1,float16,float16,0,0.024869332710901897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,64,128,1,float16,fp8,0,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,64,128,1,fp8,fp8,0,0.033146666983763375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,64,0,1,fp8,fp8,0,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,64,128,1,float16,float16,0,0.024490666886170704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,64,0,1,float16,float16,0,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,64,128,1,float16,fp8,0,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,64,128,1,fp8,fp8,0,0.03139200061559677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,64,128,1,fp8,fp8,0,0.031343999008337654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,64,0,1,float16,fp8,0,0.023215999205907185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,64,0,1,fp8,fp8,0,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,64,128,1,float16,float16,0,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,64,0,1,float16,float16,0,0.022175999979178112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,64,128,1,float16,fp8,0,0.024506665766239166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,64,0,1,float16,fp8,0,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,64,0,1,fp8,fp8,0,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,64,128,1,float16,float16,0,0.024703999360402424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,64,0,1,float16,float16,0,0.022511998812357586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,64,128,1,float16,fp8,0,0.024533333877722423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,64,128,1,fp8,fp8,0,0.030858665704727173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,64,0,1,float16,fp8,0,0.02258133391539256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,64,0,1,fp8,fp8,0,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,64,128,1,float16,float16,0,0.023760000864664715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,64,0,1,float16,float16,0,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,64,128,1,float16,fp8,0,0.0240639994541804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,64,128,1,fp8,fp8,0,0.030266667405764263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,64,0,1,float16,fp8,0,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,64,0,1,float16,fp8,0,0.02202133337656657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,64,0,1,fp8,fp8,0,0.0207893339296182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,64,128,1,float16,float16,0,0.023344000180562336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,64,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,64,128,1,float16,fp8,0,0.023941333095232647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,64,128,1,fp8,fp8,0,0.03052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,64,0,1,float16,fp8,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,64,0,1,fp8,fp8,0,0.020869334538777668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,64,128,1,float16,float16,0,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,64,0,1,float16,float16,0,0.021925332645575207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,64,128,1,float16,fp8,0,0.023925334215164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,64,128,1,fp8,fp8,0,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,64,128,1,fp8,fp8,0,0.029722665747006733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,64,0,1,fp8,fp8,0,0.020810666183630627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,64,128,1,float16,float16,0,0.02288000037272771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,64,0,1,float16,float16,0,0.02070933332045873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,64,128,1,float16,fp8,0,0.0235359991590182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,64,0,1,float16,fp8,0,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,64,0,1,fp8,fp8,0,0.019999999552965164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,64,128,1,float16,float16,0,0.022730665902296703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,64,0,1,float16,float16,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,64,128,1,float16,fp8,0,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,64,128,1,fp8,fp8,0,0.029445332785447437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,64,0,1,float16,fp8,0,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,64,0,1,fp8,fp8,0,0.020746666938066483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,64,128,1,float16,float16,0,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,64,0,1,fp8,fp8,0,0.020581333587567013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,64,0,1,float16,float16,0,0.020874666670958202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,64,128,1,float16,fp8,0,0.02366400013367335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,64,128,1,fp8,fp8,0,0.029690665503342945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,64,0,1,float16,fp8,0,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,64,128,1,float16,float16,0,0.1467519998550415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,64,0,1,float16,float16,0,0.14296533664067587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,64,128,1,float16,fp8,0,0.14510933558146158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,64,128,1,fp8,fp8,0,0.18441067139307657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,64,0,1,float16,fp8,0,0.1436906655629476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,64,0,1,fp8,fp8,0,0.13165866335233053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,64,0,1,float16,fp8,0,0.15149866541226706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,64,128,1,float16,float16,0,0.15705066919326782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,64,0,1,float16,float16,0,0.15388266245524088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,64,128,1,float16,fp8,0,0.1546346644560496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,64,128,1,fp8,fp8,0,0.18933866421381632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,64,0,1,fp8,fp8,0,0.13527466853459677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,64,128,1,float16,float16,0,0.0999679962793986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,64,0,1,float16,float16,0,0.0995146632194519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,64,128,1,float16,fp8,0,0.09752532839775085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,64,128,1,fp8,fp8,0,0.10921066999435425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,64,0,1,float16,fp8,0,0.09604799747467041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,64,0,1,fp8,fp8,0,0.0832533339659373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,64,0,1,float16,fp8,0,0.07554666697978973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,64,128,1,float16,float16,0,0.07766399780909221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,64,0,1,float16,float16,0,0.07547200222810109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,64,128,1,float16,float16,0,0.08479467034339905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,64,128,1,float16,fp8,0,0.0767680009206136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,64,128,1,fp8,fp8,0,0.1018506685892741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,64,0,1,fp8,fp8,0,0.07589333256085713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,64,0,1,float16,float16,0,0.08404266834259033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,64,128,1,float16,fp8,0,0.08326399823029836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,64,128,1,fp8,fp8,0,0.10392000277837117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,64,0,1,float16,fp8,0,0.08125333487987518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,64,0,1,fp8,fp8,0,0.077824001510938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,64,128,1,float16,float16,0,0.04901333153247833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,64,0,1,float16,float16,0,0.048298666874567665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,64,128,1,float16,fp8,0,0.04786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,64,128,1,fp8,fp8,0,0.0645066648721695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,64,0,1,float16,fp8,0,0.047824000318845115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,64,0,1,fp8,fp8,0,0.04961066444714864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,64,0,1,float16,float16,0,0.04182399809360504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,64,128,1,float16,float16,0,0.04221333563327789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,64,128,1,float16,fp8,0,0.04227200150489807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,64,128,1,fp8,fp8,0,0.05553600192070007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,64,0,1,float16,fp8,0,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,64,0,1,fp8,fp8,0,0.0431573341290156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,64,128,1,float16,float16,0,0.044490665197372437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,64,0,1,float16,float16,0,0.04409066836039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,64,128,1,float16,fp8,0,0.04347200194994608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,64,128,1,fp8,fp8,0,0.05841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,64,0,1,float16,fp8,0,0.04294399917125702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,64,0,1,fp8,fp8,0,0.04474133253097534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,64,128,1,float16,float16,0,0.034527999659379326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,64,128,1,float16,float16,0,0.031856000423431396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,64,0,1,float16,float16,0,0.031471999982992806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,64,128,1,float16,fp8,0,0.03180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,64,0,1,float16,float16,0,0.03386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,64,128,1,float16,fp8,0,0.03418133407831192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,64,0,1,fp8,fp8,0,0.02886933336655299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,64,128,1,float16,float16,0,0.032672000428040825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,64,0,1,float16,float16,0,0.032186667124430336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,64,128,1,float16,fp8,0,0.03242666771014532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,64,128,1,fp8,fp8,0,0.03658666710058848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,64,0,1,float16,fp8,0,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,64,0,1,fp8,fp8,0,0.029951999584833782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,64,128,1,fp8,fp8,0,0.03514133393764496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,64,0,1,float16,float16,0,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,64,0,1,float16,fp8,0,0.031397332747777305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,64,128,1,float16,fp8,0,0.022005334496498108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,64,128,1,fp8,fp8,0,0.035504000882307686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,64,0,1,float16,fp8,0,0.032272001107533775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,64,0,1,fp8,fp8,0,0.02897600084543228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,64,0,1,float16,float16,0,0.0200853335360686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,64,128,1,float16,float16,0,0.022304000953833263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,64,128,1,fp8,fp8,0,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,64,0,1,float16,fp8,0,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,64,0,1,fp8,fp8,0,0.021738665799299877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,64,128,1,float16,float16,0,0.020058666666348774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,64,0,1,float16,float16,0,0.020527999848127365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,64,128,1,float16,fp8,0,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,64,128,1,fp8,fp8,0,0.02446399877468745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,64,0,1,float16,fp8,0,0.020303999384244282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,64,0,1,fp8,fp8,0,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,64,128,1,float16,float16,0,0.020506666352351505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,64,128,1,float16,fp8,0,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,64,128,1,fp8,fp8,0,0.02481066683928172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,64,0,1,float16,fp8,0,0.020736000190178554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,64,0,1,float16,fp8,0,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,64,0,1,fp8,fp8,0,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,64,128,1,float16,float16,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,64,0,1,float16,float16,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,64,128,1,float16,fp8,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,64,128,1,fp8,fp8,0,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,64,0,1,fp8,fp8,0,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,64,128,1,float16,float16,0,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,64,0,1,float16,float16,0,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,64,128,1,float16,fp8,0,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,64,128,1,fp8,fp8,0,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,64,128,1,float16,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,64,0,1,float16,fp8,0,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,64,0,1,fp8,fp8,0,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,64,128,1,float16,float16,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,64,0,1,float16,float16,0,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,64,128,1,fp8,fp8,0,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,64,0,1,float16,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,64,0,1,fp8,fp8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,64,128,1,float16,float16,0,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,64,0,1,float16,float16,0,0.015728000551462173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,64,128,1,float16,fp8,0,0.015893333901961643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,64,128,1,fp8,fp8,0,0.022437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,64,0,1,float16,fp8,0,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,64,0,1,fp8,fp8,0,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,64,128,1,float16,float16,0,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,64,0,1,float16,float16,0,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,64,128,1,float16,fp8,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,64,128,1,fp8,fp8,0,0.021984001000722248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,64,0,1,float16,fp8,0,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,64,0,1,fp8,fp8,0,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,64,128,1,float16,float16,0,0.015520000209410986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,64,0,1,float16,float16,0,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,64,128,1,float16,fp8,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,64,128,1,fp8,fp8,0,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,64,0,1,float16,fp8,0,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,64,0,1,fp8,fp8,0,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,64,128,1,float16,float16,0,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,64,0,1,float16,float16,0,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,64,128,1,float16,fp8,0,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,64,128,1,fp8,fp8,0,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,64,0,1,float16,fp8,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,64,0,1,fp8,fp8,0,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,64,128,1,float16,float16,0,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,64,0,1,float16,float16,0,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,64,128,1,float16,fp8,0,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,64,128,1,fp8,fp8,0,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,64,0,1,float16,fp8,0,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,64,0,1,fp8,fp8,0,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,64,128,1,float16,float16,0,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,64,0,1,float16,float16,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,64,128,1,float16,fp8,0,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,64,128,1,fp8,fp8,0,0.021664001047611237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,64,0,1,float16,fp8,0,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,64,0,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,64,128,1,float16,float16,0,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,64,0,1,float16,float16,0,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,64,128,1,float16,fp8,0,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,64,128,1,fp8,fp8,0,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,64,0,1,float16,fp8,0,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,64,0,1,fp8,fp8,0,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,64,0,1,fp8,fp8,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,64,128,1,float16,float16,0,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,64,0,1,float16,float16,0,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,64,128,1,float16,fp8,0,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,64,128,1,fp8,fp8,0,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,64,0,1,float16,fp8,0,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,64,0,1,float16,fp8,0,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,64,128,1,float16,float16,0,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,64,0,1,float16,float16,0,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,64,128,1,float16,fp8,0,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,64,128,1,fp8,fp8,0,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,64,0,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,64,128,1,float16,float16,0,0.05868266522884369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,64,0,1,float16,float16,0,0.05948266883691152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,64,128,1,float16,fp8,0,0.059061333537101746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,64,128,1,float16,float16,0,0.06746133168538411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,64,128,1,fp8,fp8,0,0.07450133562088013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,64,0,1,float16,fp8,0,0.0595413347085317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,64,0,1,fp8,fp8,0,0.07404266794522603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,64,0,1,float16,float16,0,0.06789333124955495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,64,128,1,float16,fp8,0,0.06586666901906331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,64,128,1,fp8,fp8,0,0.07773333291212718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,64,0,1,float16,fp8,0,0.06562133133411407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,64,0,1,fp8,fp8,0,0.07711466650168101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,64,128,1,float16,float16,0,0.03984533250331879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,64,0,1,float16,float16,0,0.0391146664818128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,64,128,1,float16,fp8,0,0.03930133332808813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,64,128,1,fp8,fp8,0,0.04886933167775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,64,0,1,float16,fp8,0,0.03807466725508372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,64,128,1,fp8,fp8,0,0.0424586683511734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,64,0,1,fp8,fp8,0,0.04872000217437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,64,128,1,float16,float16,0,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,64,0,1,float16,float16,0,0.03378133227427801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,64,128,1,float16,fp8,0,0.03336533407370249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,64,0,1,float16,fp8,0,0.03376533339420954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,64,0,1,fp8,fp8,0,0.04218666752179464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,64,128,1,float16,float16,0,0.03509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,64,0,1,float16,float16,0,0.03533866753180822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,64,128,1,float16,fp8,0,0.03469866762558619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,64,0,1,float16,float16,0,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,64,128,1,fp8,fp8,0,0.044405331214269005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,64,0,1,float16,fp8,0,0.03457599878311157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,64,0,1,fp8,fp8,0,0.043807998299598694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,64,128,1,float16,float16,0,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,64,128,1,float16,float16,0,0.023215999205907185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,64,0,1,float16,float16,0,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,64,128,1,float16,fp8,0,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,64,128,1,fp8,fp8,0,0.029029332101345062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,64,0,1,float16,fp8,0,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,64,0,1,fp8,fp8,0,0.029546665648619335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,64,128,1,float16,fp8,0,0.02348800003528595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,64,128,1,fp8,fp8,0,0.027845333019892376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,64,0,1,float16,fp8,0,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,64,128,1,float16,fp8,0,0.02401600033044815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,64,128,1,fp8,fp8,0,0.02828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,64,0,1,fp8,fp8,0,0.02874133239189784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,64,128,1,float16,float16,0,0.024122667809327442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,64,0,1,float16,float16,0,0.024218666056791942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,64,0,1,float16,fp8,0,0.02382933348417282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,64,0,1,fp8,fp8,0,0.028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,64,128,1,float16,float16,0,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,64,0,1,float16,float16,0,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,64,128,1,float16,fp8,0,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,64,128,1,fp8,fp8,0,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,64,0,1,float16,fp8,0,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,64,0,1,fp8,fp8,0,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,64,128,1,float16,float16,0,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,64,0,1,float16,float16,0,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,64,128,1,float16,fp8,0,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,64,128,1,fp8,fp8,0,0.020661332954963047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,64,128,1,float16,fp8,0,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,64,0,1,float16,fp8,0,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,64,0,1,fp8,fp8,0,0.02083733429511388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,64,128,1,float16,float16,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,64,0,1,float16,float16,0,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,64,128,1,fp8,fp8,0,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,64,0,1,float16,fp8,0,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,64,0,1,fp8,fp8,0,0.02073066681623459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,64,128,1,float16,float16,0,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,64,0,1,float16,float16,0,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,64,128,1,float16,fp8,0,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,64,128,1,fp8,fp8,0,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,64,0,1,float16,fp8,0,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,64,0,1,fp8,fp8,0,0.01958400011062622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,64,0,1,float16,fp8,0,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,64,128,1,float16,float16,0,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,64,0,1,float16,float16,0,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,64,128,1,float16,fp8,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,64,128,1,fp8,fp8,0,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,64,128,1,fp8,fp8,0,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,64,0,1,fp8,fp8,0,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,64,128,1,float16,float16,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,64,0,1,float16,float16,0,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,64,128,1,float16,fp8,0,0.014618666221698126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,64,0,1,float16,fp8,0,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,64,0,1,fp8,fp8,0,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,64,128,1,float16,float16,0,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,64,0,1,float16,float16,0,0.013690666606028875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,64,128,1,float16,fp8,0,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,64,128,1,fp8,fp8,0,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,64,0,1,float16,fp8,0,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,64,0,1,fp8,fp8,0,0.017946666727463405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,64,128,1,float16,float16,0,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,64,0,1,float16,float16,0,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,64,128,1,float16,fp8,0,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,64,128,1,fp8,fp8,0,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,64,0,1,float16,fp8,0,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,64,128,1,fp8,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,64,0,1,fp8,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,64,128,1,float16,float16,0,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,64,0,1,float16,float16,0,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,64,128,1,float16,fp8,0,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,64,0,1,float16,fp8,0,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,64,0,1,fp8,fp8,0,0.0183999997874101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,64,128,1,float16,float16,0,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,64,0,1,float16,float16,0,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,64,128,1,float16,fp8,0,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,64,128,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,64,0,1,float16,fp8,0,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,64,0,1,fp8,fp8,0,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,64,128,1,float16,float16,0,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,64,0,1,float16,float16,0,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,64,128,1,float16,fp8,0,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,64,128,1,fp8,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,64,0,1,float16,fp8,0,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,64,0,1,fp8,fp8,0,0.017535999417304993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,64,128,1,float16,float16,0,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,64,0,1,float16,float16,0,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,64,0,1,float16,float16,0,0.012522666404644648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,64,128,1,float16,fp8,0,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,64,128,1,fp8,fp8,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,64,0,1,float16,fp8,0,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,64,0,1,fp8,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,64,128,1,float16,float16,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,64,128,1,float16,fp8,0,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,64,128,1,fp8,fp8,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,64,0,1,float16,fp8,0,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,64,0,1,fp8,fp8,0,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,64,128,1,float16,float16,0,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,64,0,1,float16,float16,0,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,64,128,1,float16,fp8,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,64,128,1,fp8,fp8,0,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,64,0,1,float16,fp8,0,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,64,0,1,fp8,fp8,0,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,64,128,1,float16,float16,0,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,64,0,1,float16,float16,0,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,64,128,1,float16,fp8,0,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,64,128,1,fp8,fp8,0,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,64,0,1,float16,fp8,0,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,64,0,1,fp8,fp8,0,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,64,128,1,float16,float16,0,0.012485332787036896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,64,0,1,float16,float16,0,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,64,128,1,float16,fp8,0,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,64,128,1,fp8,fp8,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,64,0,1,float16,fp8,0,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,64,0,1,fp8,fp8,0,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,64,128,1,float16,float16,0,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,64,0,1,float16,float16,0,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,64,128,1,float16,fp8,0,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,64,128,1,fp8,fp8,0,0.017701332767804463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,64,0,1,float16,fp8,0,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,64,0,1,fp8,fp8,0,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,64,128,1,float16,float16,0,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,64,0,1,float16,float16,0,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,64,128,1,float16,fp8,0,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,64,128,1,fp8,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,64,0,1,float16,fp8,0,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,64,0,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,64,128,1,float16,float16,0,0.03310399999221166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,64,0,1,float16,float16,0,0.03243733445803324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,64,128,1,float16,fp8,0,0.03224000086386999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,64,128,1,fp8,fp8,0,0.053557331363360085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,64,0,1,float16,fp8,0,0.032698666055997215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,64,128,1,float16,float16,0,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,64,0,1,fp8,fp8,0,0.05386666456858317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,64,0,1,float16,float16,0,0.033941333492596946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,64,128,1,float16,fp8,0,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,64,0,1,float16,fp8,0,0.03323733309904734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,64,128,1,fp8,fp8,0,0.054805333415667214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,64,128,1,fp8,fp8,0,0.03451200077931086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,64,128,1,float16,float16,0,0.02292799949645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,64,0,1,float16,fp8,0,0.02272533377011617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,64,0,1,fp8,fp8,0,0.055760001142819725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,64,0,1,float16,float16,0,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,64,128,1,float16,fp8,0,0.022672000030676525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,64,0,1,fp8,fp8,0,0.03454933315515518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,64,128,1,float16,float16,0,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,64,0,1,float16,fp8,0,0.02163200080394745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,64,0,1,float16,float16,0,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,64,128,1,float16,fp8,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,64,128,1,fp8,fp8,0,0.03408533334732056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,64,0,1,fp8,fp8,0,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,64,128,1,float16,float16,0,0.022074667116006214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,64,0,1,float16,float16,0,0.02232533444960912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,64,0,1,fp8,fp8,0,0.034314667185147606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,64,128,1,float16,float16,0,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,64,128,1,float16,fp8,0,0.022074667116006214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,64,128,1,fp8,fp8,0,0.03421866645415624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,64,0,1,float16,fp8,0,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,64,0,1,float16,float16,0,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,64,128,1,float16,fp8,0,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,64,128,1,fp8,fp8,0,0.024853333830833435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,64,0,1,float16,fp8,0,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,64,0,1,fp8,fp8,0,0.024864000578721363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,64,128,1,float16,float16,0,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,64,0,1,float16,float16,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,64,128,1,float16,fp8,0,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,64,128,1,fp8,fp8,0,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,64,0,1,float16,fp8,0,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,64,0,1,fp8,fp8,0,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,64,128,1,float16,float16,0,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,64,0,1,float16,float16,0,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,64,128,1,float16,fp8,0,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,64,128,1,fp8,fp8,0,0.024133334557215374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,64,0,1,float16,fp8,0,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,64,0,1,fp8,fp8,0,0.023957334458827972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,64,128,1,float16,float16,0,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,64,0,1,float16,float16,0,0.013573333621025085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,64,128,1,float16,fp8,0,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,64,128,1,fp8,fp8,0,0.019578666736682255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,64,0,1,float16,fp8,0,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,64,128,1,float16,float16,0,0.013562666873137156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,64,0,1,fp8,fp8,0,0.019760000209013622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,64,0,1,float16,float16,0,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,64,128,1,float16,fp8,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,64,128,1,fp8,fp8,0,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,64,128,1,fp8,fp8,0,0.019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,64,0,1,float16,fp8,0,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,64,0,1,fp8,fp8,0,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,64,128,1,float16,float16,0,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,64,0,1,float16,float16,0,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,64,128,1,float16,fp8,0,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,64,0,1,float16,fp8,0,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,64,0,1,float16,fp8,0,0.012533333152532578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,64,0,1,fp8,fp8,0,0.019600000232458115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,64,128,1,float16,float16,0,0.012319999436537424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,64,0,1,float16,float16,0,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,64,128,1,float16,fp8,0,0.012175999581813812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,64,128,1,fp8,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,64,128,1,fp8,fp8,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,64,0,1,fp8,fp8,0,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,64,128,1,float16,float16,0,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,64,0,1,float16,float16,0,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,64,128,1,float16,fp8,0,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,64,0,1,float16,fp8,0,0.01227733368674914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,64,0,1,fp8,fp8,0,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,64,128,1,float16,float16,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,64,0,1,float16,float16,0,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,64,128,1,float16,fp8,0,0.0124746672809124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,64,128,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,64,0,1,float16,fp8,0,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,64,0,1,fp8,fp8,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,64,128,1,float16,float16,0,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,64,0,1,float16,float16,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,64,128,1,float16,fp8,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,64,0,1,float16,fp8,0,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,64,128,1,fp8,fp8,0,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,64,0,1,fp8,fp8,0,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,64,128,1,float16,float16,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,64,0,1,fp8,fp8,0,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,64,0,1,float16,float16,0,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,64,128,1,float16,fp8,0,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,64,128,1,fp8,fp8,0,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,64,128,1,fp8,fp8,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,64,0,1,float16,fp8,0,0.012026666353146235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,64,128,1,float16,float16,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,64,0,1,float16,float16,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,64,128,1,float16,fp8,0,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,64,0,1,float16,fp8,0,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,64,0,1,fp8,fp8,0,0.018165333817402523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,64,128,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,64,0,1,fp8,fp8,0,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,64,0,1,float16,float16,0,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,64,128,1,float16,fp8,0,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,64,128,1,fp8,fp8,0,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,64,0,1,float16,fp8,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,64,128,1,float16,float16,0,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,64,0,1,float16,float16,0,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,64,128,1,float16,fp8,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,64,128,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,64,0,1,float16,fp8,0,0.011930666863918304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,64,0,1,fp8,fp8,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,64,128,1,float16,float16,0,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,64,0,1,float16,float16,0,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,64,128,1,float16,fp8,0,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,64,128,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,64,0,1,float16,fp8,0,0.012250666817029318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,64,0,1,fp8,fp8,0,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,64,128,1,float16,float16,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,64,0,1,float16,float16,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,64,128,1,float16,fp8,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,64,128,1,fp8,fp8,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,64,0,1,float16,fp8,0,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,64,0,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,64,128,1,float16,float16,0,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,64,0,1,float16,float16,0,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,64,128,1,float16,fp8,0,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,64,128,1,fp8,fp8,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,64,0,1,float16,fp8,0,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,64,0,1,fp8,fp8,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,64,128,1,float16,float16,0,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,64,0,1,float16,float16,0,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,64,128,1,float16,fp8,0,0.012080000092585882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,64,128,1,fp8,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,64,0,1,float16,fp8,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,64,0,1,fp8,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,64,128,1,float16,float16,0,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,64,0,1,float16,float16,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,64,128,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,64,128,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,64,0,1,fp8,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,64,128,1,float16,float16,0,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,64,0,1,float16,float16,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,64,128,1,float16,fp8,0,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,64,128,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,64,0,1,float16,fp8,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,64,0,1,fp8,fp8,0,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,64,128,1,float16,float16,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,64,0,1,float16,float16,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,64,128,1,float16,fp8,0,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,64,128,1,fp8,fp8,0,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,64,0,1,float16,fp8,0,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,64,0,1,fp8,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,64,128,1,float16,float16,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,64,0,1,float16,float16,0,0.023215999205907185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,64,128,1,float16,fp8,0,0.022469334304332733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,64,128,1,fp8,fp8,0,0.0473333348830541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,64,0,1,float16,fp8,0,0.02237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,64,0,1,fp8,fp8,0,0.04684799909591675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,64,128,1,float16,float16,0,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,64,0,1,float16,float16,0,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,64,128,1,float16,fp8,0,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,64,128,1,fp8,fp8,0,0.047456001242001854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,64,0,1,float16,fp8,0,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,64,0,1,fp8,fp8,0,0.04728533327579498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,64,128,1,float16,float16,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,64,0,1,float16,float16,0,0.01605333387851715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,64,128,1,float16,fp8,0,0.016704000532627106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,64,128,1,fp8,fp8,0,0.030970667799313862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,64,0,1,float16,fp8,0,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,64,0,1,fp8,fp8,0,0.030805334448814392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,64,128,1,float16,float16,0,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,64,0,1,float16,float16,0,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,64,0,1,fp8,fp8,0,0.030031998952229817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,64,128,1,float16,fp8,0,0.01584533353646596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,64,128,1,fp8,fp8,0,0.03017599880695343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,64,0,1,float16,fp8,0,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,64,128,1,float16,float16,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,64,0,1,float16,float16,0,0.016202667107184727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,64,128,1,float16,fp8,0,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,64,128,1,fp8,fp8,0,0.03035733352104823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,64,0,1,float16,fp8,0,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,64,0,1,fp8,fp8,0,0.03046400099992752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,64,128,1,float16,float16,0,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,64,0,1,float16,float16,0,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,64,128,1,float16,fp8,0,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,64,128,1,fp8,fp8,0,0.022597332795461018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,64,0,1,float16,fp8,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,64,0,1,fp8,fp8,0,0.022885332504908245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,64,128,1,float16,float16,0,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,64,0,1,float16,float16,0,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,64,128,1,float16,fp8,0,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,64,128,1,fp8,fp8,0,0.02233600119749705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,64,0,1,float16,fp8,0,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,64,0,1,fp8,fp8,0,0.022143999735514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,64,128,1,float16,float16,0,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,64,0,1,float16,float16,0,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,64,128,1,float16,fp8,0,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,64,128,1,fp8,fp8,0,0.0229120006163915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,64,0,1,float16,fp8,0,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,64,0,1,fp8,fp8,0,0.022848000129063923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,64,128,1,float16,float16,0,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,64,0,1,float16,float16,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,64,128,1,float16,fp8,0,0.012240000069141388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,64,128,1,fp8,fp8,0,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,64,0,1,float16,fp8,0,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,64,0,1,fp8,fp8,0,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,64,128,1,float16,float16,0,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,64,0,1,float16,float16,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,64,0,1,fp8,fp8,0,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,64,128,1,float16,fp8,0,0.01221866657336553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,64,128,1,fp8,fp8,0,0.018645333747069042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,64,0,1,float16,fp8,0,0.012485332787036896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,64,128,1,float16,float16,0,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,64,0,1,float16,float16,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,64,128,1,float16,fp8,0,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,64,128,1,fp8,fp8,0,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,64,0,1,float16,fp8,0,0.012234666695197424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,64,0,1,fp8,fp8,0,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,64,128,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,64,0,1,float16,float16,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,64,128,1,float16,fp8,0,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,64,128,1,fp8,fp8,0,0.01773333301146825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,64,128,1,fp8,fp8,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,64,0,1,float16,fp8,0,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,64,0,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,64,128,1,float16,float16,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,64,0,1,float16,float16,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,64,128,1,float16,fp8,0,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,64,0,1,float16,fp8,0,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,64,0,1,fp8,fp8,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,64,128,1,float16,float16,0,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,64,0,1,float16,float16,0,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,64,128,1,float16,fp8,0,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,64,128,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,64,0,1,float16,fp8,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,64,0,1,fp8,fp8,0,0.018058666338523228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,64,128,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,64,0,1,float16,float16,0,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,64,128,1,float16,fp8,0,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,64,128,1,float16,fp8,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,64,128,1,fp8,fp8,0,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,64,0,1,float16,fp8,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,64,0,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,64,128,1,float16,float16,0,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,64,0,1,float16,float16,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,64,0,1,float16,float16,0,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,64,128,1,fp8,fp8,0,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,64,0,1,float16,fp8,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,64,0,1,fp8,fp8,0,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,64,128,1,float16,float16,0,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,64,128,1,float16,fp8,0,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,64,128,1,fp8,fp8,0,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,64,0,1,float16,fp8,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,64,0,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,64,128,1,float16,float16,0,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,64,0,1,float16,float16,0,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,64,128,1,float16,fp8,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,64,128,1,fp8,fp8,0,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,64,0,1,float16,fp8,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,64,0,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,64,128,1,float16,float16,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,64,0,1,float16,float16,0,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,64,128,1,float16,fp8,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,64,128,1,fp8,fp8,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,64,0,1,float16,fp8,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,64,0,1,fp8,fp8,0,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,64,128,1,float16,float16,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,64,0,1,float16,float16,0,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,64,128,1,float16,fp8,0,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,64,128,1,fp8,fp8,0,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,64,0,1,float16,fp8,0,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,64,0,1,fp8,fp8,0,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,64,128,1,float16,float16,0,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,64,0,1,float16,float16,0,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,64,128,1,float16,fp8,0,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,64,128,1,fp8,fp8,0,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,64,0,1,float16,fp8,0,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,64,0,1,fp8,fp8,0,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,64,128,1,float16,float16,0,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,64,0,1,float16,float16,0,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,64,128,1,float16,fp8,0,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,64,128,1,fp8,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,64,0,1,float16,fp8,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,64,128,1,fp8,fp8,0,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,64,0,1,fp8,fp8,0,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,64,128,1,float16,float16,0,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,64,0,1,float16,float16,0,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,64,128,1,float16,fp8,0,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,64,0,1,float16,fp8,0,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,64,0,1,fp8,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,64,128,1,float16,float16,0,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,64,0,1,float16,float16,0,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,64,128,1,float16,fp8,0,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,64,128,1,fp8,fp8,0,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,64,0,1,float16,fp8,0,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,64,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,64,128,1,float16,float16,0,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,64,0,1,float16,float16,0,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,64,128,1,float16,fp8,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,64,128,1,fp8,fp8,0,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,64,0,1,float16,fp8,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,64,0,1,fp8,fp8,0,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,64,128,1,float16,float16,0,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,64,0,1,float16,float16,0,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,64,128,1,float16,fp8,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,64,128,1,fp8,fp8,0,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,64,0,1,float16,fp8,0,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,64,0,1,fp8,fp8,0,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,64,128,1,float16,float16,0,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,64,0,1,float16,float16,0,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,64,128,1,float16,fp8,0,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,64,128,1,fp8,fp8,0,0.04422933359940847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,64,0,1,float16,fp8,0,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,64,0,1,fp8,fp8,0,0.043381333351135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,64,128,1,float16,float16,0,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,64,0,1,float16,float16,0,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,64,128,1,float16,fp8,0,0.019551999866962433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,64,128,1,fp8,fp8,0,0.0444213350613912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,64,0,1,float16,fp8,0,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,64,0,1,fp8,fp8,0,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,64,128,1,float16,float16,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,64,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,64,128,1,float16,fp8,0,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,64,128,1,fp8,fp8,0,0.028901333610216778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,64,0,1,float16,fp8,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,64,0,1,fp8,fp8,0,0.02940800040960312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,64,128,1,float16,float16,0,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,64,0,1,float16,float16,0,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,64,128,1,float16,fp8,0,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,64,128,1,fp8,fp8,0,0.028677334388097126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,64,0,1,float16,fp8,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,64,0,1,fp8,fp8,0,0.028581333657105763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,64,128,1,float16,float16,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,64,0,1,float16,float16,0,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,64,128,1,float16,fp8,0,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,64,128,1,float16,fp8,0,0.012335999558369318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,64,128,1,fp8,fp8,0,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,64,0,1,float16,fp8,0,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,64,0,1,fp8,fp8,0,0.029098667204380035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,64,128,1,float16,float16,0,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,64,0,1,float16,float16,0,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,64,128,1,fp8,fp8,0,0.02219199885924657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,64,0,1,float16,fp8,0,0.012298667182525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,64,0,1,fp8,fp8,0,0.02197866638501485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,64,0,1,fp8,fp8,0,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,64,128,1,float16,float16,0,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,64,0,1,float16,float16,0,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,64,128,1,float16,fp8,0,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,64,0,1,float16,fp8,0,0.012629333883523941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,64,128,1,fp8,fp8,0,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,64,128,1,float16,float16,0,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,64,128,1,float16,fp8,0,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,64,0,1,float16,float16,0,0.012346666306257248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,64,128,1,fp8,fp8,0,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,64,0,1,float16,fp8,0,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,64,0,1,fp8,fp8,0,0.021984001000722248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,64,128,1,float16,float16,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,64,0,1,float16,float16,0,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,64,128,1,float16,fp8,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,64,128,1,fp8,fp8,0,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,64,0,1,float16,fp8,0,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,64,0,1,fp8,fp8,0,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,64,128,1,float16,float16,0,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,64,0,1,float16,float16,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,64,128,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,64,128,1,fp8,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,64,0,1,float16,fp8,0,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,64,0,1,fp8,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,64,128,1,float16,float16,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,64,0,1,float16,float16,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,64,128,1,float16,fp8,0,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,64,128,1,fp8,fp8,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,64,0,1,float16,fp8,0,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,64,0,1,fp8,fp8,0,0.018432000031073887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,64,128,1,float16,float16,0,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,64,0,1,float16,float16,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,64,128,1,float16,fp8,0,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,64,128,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,64,0,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,64,0,1,fp8,fp8,0,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,64,128,1,float16,float16,0,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,64,0,1,float16,float16,0,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,64,128,1,float16,fp8,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,64,128,1,fp8,fp8,0,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,64,0,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,64,0,1,fp8,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,64,128,1,float16,float16,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,64,0,1,float16,float16,0,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,64,128,1,float16,fp8,0,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,64,128,1,fp8,fp8,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,64,0,1,float16,fp8,0,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,64,0,1,fp8,fp8,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,64,128,1,float16,float16,0,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,64,0,1,float16,float16,0,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,64,128,1,float16,fp8,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,64,128,1,fp8,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,64,0,1,float16,fp8,0,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,64,0,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,64,128,1,float16,float16,0,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,64,0,1,float16,float16,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,64,128,1,float16,fp8,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,64,128,1,fp8,fp8,0,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,64,0,1,float16,fp8,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,64,0,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,64,128,1,float16,float16,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,64,0,1,float16,float16,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,64,128,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,64,128,1,fp8,fp8,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,64,0,1,float16,fp8,0,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,64,128,1,fp8,fp8,0,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,64,0,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,64,128,1,float16,float16,0,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,64,0,1,float16,float16,0,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,64,128,1,float16,fp8,0,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,64,0,1,float16,fp8,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,64,0,1,fp8,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,64,128,1,float16,float16,0,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,64,0,1,float16,float16,0,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,64,128,1,float16,fp8,0,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,64,128,1,fp8,fp8,0,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,64,0,1,float16,fp8,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,64,0,1,fp8,fp8,0,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,64,128,1,float16,float16,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,64,128,1,float16,float16,0,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,64,0,1,float16,float16,0,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,64,128,1,float16,fp8,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,64,128,1,fp8,fp8,0,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,64,128,1,fp8,fp8,0,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,64,0,1,float16,fp8,0,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,64,0,1,fp8,fp8,0,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,64,0,1,float16,float16,0,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,64,128,1,float16,fp8,0,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,64,0,1,float16,fp8,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,64,0,1,fp8,fp8,0,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,64,128,1,float16,float16,0,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,64,0,1,float16,float16,0,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,64,128,1,float16,fp8,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,64,128,1,fp8,fp8,0,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,64,0,1,float16,fp8,0,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,64,0,1,fp8,fp8,0,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,64,128,1,float16,float16,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,64,0,1,float16,float16,0,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,64,128,1,float16,fp8,0,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,64,128,1,fp8,fp8,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,64,0,1,float16,fp8,0,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,64,0,1,float16,fp8,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,64,0,1,fp8,fp8,0,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,64,128,1,float16,float16,0,0.010319999729593595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,64,0,1,float16,float16,0,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,64,128,1,float16,fp8,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,64,128,1,fp8,fp8,0,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,64,0,1,float16,fp8,0,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,64,0,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,64,128,1,float16,float16,0,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,64,128,1,float16,fp8,0,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,64,0,1,float16,float16,0,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,64,128,1,float16,fp8,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,64,128,1,fp8,fp8,0,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,64,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,64,128,1,float16,float16,0,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,64,0,1,float16,float16,0,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,64,128,1,fp8,fp8,0,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,64,128,1,float16,fp8,0,0.21300800641377768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,64,0,1,float16,fp8,0,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,64,0,1,fp8,fp8,0,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,64,128,1,float16,float16,0,0.21596266825993857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,64,128,1,fp8,fp8,0,0.2789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,64,128,1,float16,float16,0,0.13079466422398886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,64,0,1,float16,float16,0,1.276863972345988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,64,128,1,float16,fp8,0,0.12931733330090842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,64,128,1,fp8,fp8,0,0.17126933733622232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,64,0,1,float16,float16,0,0.617845336596171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,64,0,1,fp8,fp8,0,1.0952746868133545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,64,0,1,float16,fp8,0,1.2690133253733318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,64,0,1,float16,fp8,0,0.6170719861984253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,64,128,1,float16,float16,0,0.11918933192888896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,64,0,1,fp8,fp8,0,0.5723626613616943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,64,128,1,float16,fp8,0,0.11635733644167583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,64,0,1,float16,float16,0,0.6081813176472982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,64,128,1,fp8,fp8,0,0.16285866498947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,64,128,1,float16,float16,0,0.07620266576608022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,64,0,1,float16,float16,0,0.31651200850804645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,64,128,1,float16,fp8,0,0.0759680022795995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,64,0,1,fp8,fp8,0,0.5635999838511149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,64,0,1,float16,fp8,0,0.6068586508433024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,64,128,1,fp8,fp8,0,0.10365333159764607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,64,128,1,float16,float16,0,0.07258133093516032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,64,0,1,fp8,fp8,0,0.30160532395044964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,64,0,1,float16,fp8,0,0.3187040090560913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,64,0,1,float16,float16,0,0.3165280024210612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,64,128,1,float16,fp8,0,0.07273066540559132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,64,128,1,fp8,fp8,0,0.09010666608810425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,64,0,1,float16,fp8,0,0.315008004506429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,64,128,1,float16,float16,0,0.05532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,64,0,1,fp8,fp8,0,0.2955519954363505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,64,0,1,float16,float16,0,0.17573867241541544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,64,128,1,float16,fp8,0,0.055589333176612854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,64,128,1,fp8,fp8,0,0.06753600140412648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,64,0,1,float16,fp8,0,0.17435733477274576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,64,128,1,float16,float16,0,0.05533333122730255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,64,0,1,fp8,fp8,0,0.1646986703077952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,64,0,1,float16,float16,0,0.1750346620877584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,64,128,1,float16,fp8,0,0.055445333321889244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,64,128,1,float16,float16,0,0.1674506664276123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,64,128,1,fp8,fp8,0,0.06748799979686737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,64,0,1,float16,fp8,0,0.17548267046610513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,64,0,1,fp8,fp8,0,0.1644159952799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,64,0,1,float16,float16,0,0.7070773442586263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,64,128,1,float16,fp8,0,0.16431466738382974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,64,128,1,fp8,fp8,0,0.21793067455291748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,64,0,1,float16,fp8,0,0.7083573341369629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,64,128,1,float16,float16,0,0.10105599959691365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,64,128,1,float16,fp8,0,0.09938666224479675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,64,128,1,fp8,fp8,0,0.13802666465441385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,64,0,1,float16,float16,0,0.3659040133158366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,64,0,1,fp8,fp8,0,0.6494293212890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,64,128,1,float16,fp8,0,0.09242666761080424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,64,0,1,float16,fp8,0,0.3680746555328369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,64,0,1,fp8,fp8,0,0.3470613161722819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,64,128,1,float16,float16,0,0.09346133470535278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,64,0,1,float16,float16,0,0.3622879981994629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,64,128,1,fp8,fp8,0,0.1283199985822042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,64,0,1,fp8,fp8,0,0.33790401617685956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,64,0,1,float16,fp8,0,0.360154668490092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,64,128,1,float16,float16,0,0.0609440008799235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,64,128,1,float16,fp8,0,0.06054399907588959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,64,0,1,float16,float16,0,0.2076373298962911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,64,128,1,fp8,fp8,0,0.08443199594815572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,64,0,1,float16,fp8,0,0.20925867557525635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,64,0,1,fp8,fp8,0,0.19337066014607748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,64,128,1,float16,float16,0,0.05830933153629303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,64,0,1,float16,fp8,0,0.20475733280181885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,64,0,1,float16,float16,0,0.204858660697937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,64,128,1,float16,fp8,0,0.046207999189694725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,64,128,1,float16,fp8,0,0.05809600154558817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,64,128,1,fp8,fp8,0,0.0749066670735677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,64,0,1,fp8,fp8,0,0.19006399313608804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,64,128,1,float16,float16,0,0.04747200012207031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,64,0,1,float16,float16,0,0.13455466429392496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,64,128,1,fp8,fp8,0,0.051632001996040344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,64,0,1,fp8,fp8,0,0.1285653313000997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,64,128,1,fp8,fp8,0,0.05146133402983347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,64,0,1,float16,fp8,0,0.1346453328927358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,64,128,1,float16,float16,0,0.04964800179004669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,64,0,1,float16,float16,0,0.13635733723640442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,64,128,1,float16,fp8,0,0.04891733328501383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,64,0,1,float16,fp8,0,0.13397333025932312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,64,0,1,fp8,fp8,0,0.12847466270128885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,64,128,1,float16,float16,0,0.1409066617488861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,64,128,1,float16,fp8,0,0.1376213332017263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,64,128,1,fp8,fp8,0,0.18898133436838785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,64,0,1,float16,float16,0,0.5123840173085531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,64,128,1,float16,float16,0,0.08447466293970744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,64,0,1,float16,fp8,0,0.5069706837336222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,64,128,1,float16,fp8,0,0.0853760043780009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,64,0,1,fp8,fp8,0,0.2583093245824178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,64,0,1,fp8,fp8,0,0.47014931837717694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,64,0,1,float16,float16,0,0.2701706687609355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,64,128,1,fp8,fp8,0,0.11919466654459636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,64,0,1,float16,fp8,0,0.27145065863927204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,64,128,1,float16,float16,0,0.07937066753705342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,64,128,1,float16,fp8,0,0.07900266846021016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,64,0,1,float16,float16,0,0.2670560081799825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,64,128,1,fp8,fp8,0,0.11028266946474712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,64,128,1,float16,float16,0,0.052602668603261314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,64,0,1,float16,fp8,0,0.2661813298861186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,64,0,1,fp8,fp8,0,0.24869332710901895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,64,0,1,float16,float16,0,0.15241600076357523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,64,128,1,float16,fp8,0,0.05381333331267039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,64,128,1,fp8,fp8,0,0.06656533479690552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,64,0,1,float16,fp8,0,0.15184533596038818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,64,0,1,fp8,fp8,0,0.14386133352915445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,64,128,1,float16,float16,0,0.0514933317899704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,64,0,1,float16,float16,0,0.15145599842071533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,64,128,1,float16,fp8,0,0.05259199937184652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,64,128,1,fp8,fp8,0,0.06385600070158641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,64,0,1,float16,fp8,0,0.15079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,64,0,1,fp8,fp8,0,0.14124799768129984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,64,0,1,float16,fp8,0,0.11493333180745442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,64,128,1,float16,float16,0,0.04453866680463155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,64,0,1,float16,float16,0,0.11504000425338745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,64,128,1,float16,fp8,0,0.04386133452256521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,64,128,1,fp8,fp8,0,0.04852266609668732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,64,128,1,float16,fp8,0,0.042549331982930504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,64,0,1,fp8,fp8,0,0.11095999677975972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,64,128,1,float16,float16,0,0.04205866654713949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,64,0,1,float16,float16,0,0.11478933691978455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,64,128,1,fp8,fp8,0,0.0481333335240682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,64,0,1,float16,fp8,0,0.11545067032178243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,64,0,1,fp8,fp8,0,0.10958400368690491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,64,128,1,float16,fp8,0,0.2099519968032837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,64,128,1,float16,float16,0,0.2138986587524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,64,128,1,fp8,fp8,0,0.2768266598383586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,64,0,1,float16,float16,0,0.7129973570505778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,64,128,1,float16,float16,0,0.1283519963423411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,64,0,1,float16,fp8,0,0.7143786748250326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,64,0,1,float16,float16,0,0.3552480141321818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,64,128,1,float16,fp8,0,0.12615999579429626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,64,0,1,fp8,fp8,0,0.6202506621678671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,64,128,1,fp8,fp8,0,0.1681119998296102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,64,0,1,float16,fp8,0,0.3545813163121541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,64,0,1,fp8,fp8,0,0.333024005095164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,64,128,1,float16,float16,0,0.11758933464686076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,64,0,1,float16,float16,0,0.3447306553522746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,64,128,1,float16,fp8,0,0.11389866471290588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,64,0,1,float16,fp8,0,0.3410773277282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,64,128,1,fp8,fp8,0,0.15806399782498678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,64,128,1,float16,float16,0,0.07217066486676534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,64,0,1,fp8,fp8,0,0.32357333103815716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,64,128,1,float16,fp8,0,0.07229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,64,0,1,float16,float16,0,0.18479466438293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,64,128,1,fp8,fp8,0,0.10122133294741313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,64,0,1,float16,fp8,0,0.18389334281285605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,64,0,1,fp8,fp8,0,0.17598400513331094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,64,128,1,float16,float16,0,0.06736533343791962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,64,0,1,float16,float16,0,0.17924267053604126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,64,0,1,fp8,fp8,0,0.17138665914535522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,64,128,1,float16,fp8,0,0.06756266454855601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,64,128,1,fp8,fp8,0,0.0942026674747467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,64,0,1,float16,fp8,0,0.18094933032989502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,64,128,1,float16,float16,0,0.04612799982229868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,64,128,1,float16,fp8,0,0.04658666749795278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,64,0,1,float16,float16,0,0.09897067149480183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,64,128,1,fp8,fp8,0,0.059530665477116905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,64,0,1,float16,fp8,0,0.10075733065605164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,64,0,1,fp8,fp8,0,0.0955466628074646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,64,128,1,float16,float16,0,0.03445333242416382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,64,128,1,float16,float16,0,0.04490133126576742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,64,0,1,float16,float16,0,0.1006773312886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,64,128,1,float16,fp8,0,0.045370668172836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,64,128,1,fp8,fp8,0,0.041375999649365745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,64,128,1,fp8,fp8,0,0.05625600119431814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,64,0,1,float16,fp8,0,0.09981866677602132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,64,0,1,fp8,fp8,0,0.09335466225941975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,64,128,1,float16,fp8,0,0.03482133398453394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,64,0,1,float16,float16,0,0.09522666533788045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,64,0,1,float16,fp8,0,0.09610133369763692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,64,0,1,fp8,fp8,0,0.0913759966691335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,64,128,1,float16,float16,0,0.03457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,64,0,1,float16,float16,0,0.09556266665458679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,64,128,1,float16,fp8,0,0.035232000052928925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,64,128,1,fp8,fp8,0,0.040463998913764954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,64,0,1,float16,fp8,0,0.09545066952705383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,64,0,1,fp8,fp8,0,0.0918346643447876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,64,128,1,float16,float16,0,0.165093332529068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,64,128,1,float16,fp8,0,0.15984533230463663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,64,0,1,float16,float16,0,0.40622933705647785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,64,128,1,fp8,fp8,0,0.21604265769322714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,64,0,1,float16,fp8,0,0.4052426815032959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,64,0,1,fp8,fp8,0,0.38066665331522626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,64,128,1,float16,float16,0,0.0972053309281667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,64,0,1,float16,float16,0,0.2179093360900879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,64,128,1,float16,fp8,0,0.09876799583435059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,64,128,1,float16,fp8,0,0.08981866637865703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,64,128,1,fp8,fp8,0,0.13727999726931253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,64,128,1,fp8,fp8,0,0.12668800354003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,64,0,1,float16,fp8,0,0.216922660668691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,64,0,1,float16,float16,0,0.12091199556986491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,64,0,1,fp8,fp8,0,0.2097866733868917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,64,128,1,float16,float16,0,0.09111467003822327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,64,0,1,float16,float16,0,0.21125332514444986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,64,0,1,float16,fp8,0,0.21517866849899292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,64,0,1,fp8,fp8,0,0.2002399961153666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,64,128,1,float16,float16,0,0.05657066901524862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,64,128,1,float16,fp8,0,0.056645333766937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,64,128,1,fp8,fp8,0,0.08144533137480418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,64,0,1,float16,fp8,0,0.12185600399971008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,64,0,1,fp8,fp8,0,0.11801600456237793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,64,128,1,float16,float16,0,0.05261866748332977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,64,0,1,float16,float16,0,0.12018666664759318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,64,128,1,float16,fp8,0,0.05377600093682607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,64,128,1,fp8,fp8,0,0.07197866837183635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,64,0,1,float16,fp8,0,0.11947733163833618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,64,0,1,fp8,fp8,0,0.11338133613268535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,64,128,1,float16,float16,0,0.04173333446184794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,64,128,1,float16,fp8,0,0.04204800228277842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,64,0,1,float16,float16,0,0.07618133227030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,64,128,1,fp8,fp8,0,0.04450133442878723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,64,0,1,float16,fp8,0,0.07622399926185608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,64,0,1,fp8,fp8,0,0.0744053324063619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,64,128,1,float16,float16,0,0.04203199843565623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,64,0,1,float16,float16,0,0.07613866527875264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,64,128,1,float16,fp8,0,0.04092800120512644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,64,128,1,fp8,fp8,0,0.04391466577847799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,64,0,1,float16,fp8,0,0.07646400233109792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,64,0,1,fp8,fp8,0,0.07369600236415863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,64,128,1,float16,float16,0,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,64,0,1,float16,float16,0,0.07538133362929027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,64,128,1,float16,fp8,0,0.03169599920511246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,64,128,1,fp8,fp8,0,0.037989333271980286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,64,128,1,float16,fp8,0,0.03173866619666418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,64,0,1,float16,fp8,0,0.07579733431339264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,64,0,1,fp8,fp8,0,0.07378133138020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,64,128,1,float16,float16,0,0.031045332551002502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,64,0,1,float16,float16,0,0.07587733368078868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,64,128,1,fp8,fp8,0,0.03802666564782461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,64,0,1,float16,fp8,0,0.07601599891980489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,64,0,1,fp8,fp8,0,0.07341333230336507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,64,128,1,float16,float16,0,0.21626132726669312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,64,128,1,float16,fp8,0,0.21195199092229208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,64,0,1,float16,float16,0,0.43381333351135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,64,128,1,fp8,fp8,0,0.2747199932734172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,64,128,1,float16,float16,0,0.1301866670449575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,64,0,1,float16,fp8,0,0.4300373395284017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,64,0,1,fp8,fp8,0,0.38654931386311847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,64,0,1,float16,float16,0,0.22499199708302817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,64,128,1,float16,fp8,0,0.12626133362452188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,64,128,1,fp8,fp8,0,0.1667840083440145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,64,0,1,float16,fp8,0,0.2229599952697754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,64,0,1,fp8,fp8,0,0.21292267243067423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,64,128,1,float16,float16,0,0.11748266220092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,64,0,1,float16,float16,0,0.21282132466634116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,64,128,1,float16,fp8,0,0.11514666676521301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,64,128,1,fp8,fp8,0,0.15832533439000449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,64,0,1,float16,fp8,0,0.21102933088938394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,64,0,1,fp8,fp8,0,0.20279999574025473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,64,128,1,float16,float16,0,0.07088000078996022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,64,0,1,float16,float16,0,0.11743467052777608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,64,128,1,float16,fp8,0,0.0718399981657664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,64,128,1,float16,fp8,0,0.06613866488138835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,64,128,1,fp8,fp8,0,0.10060800115267436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,64,0,1,float16,fp8,0,0.11809066931406657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,64,0,1,fp8,fp8,0,0.11508799592653911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,64,128,1,float16,float16,0,0.06622399886449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,64,0,1,float16,float16,0,0.11424533526102702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,64,128,1,fp8,fp8,0,0.09358400106430054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,64,0,1,float16,fp8,0,0.11341866850852966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,64,0,1,fp8,fp8,0,0.10734400153160095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,64,128,1,float16,float16,0,0.043552001317342125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,64,0,1,float16,float16,0,0.06568533182144165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,64,128,1,float16,fp8,0,0.043765331308046974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,64,128,1,fp8,fp8,0,0.056133334835370384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,64,0,1,float16,fp8,0,0.06623999774456024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,64,0,1,fp8,fp8,0,0.06435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,64,128,1,float16,float16,0,0.04274133344491323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,64,0,1,float16,float16,0,0.0652213344971339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,64,128,1,float16,fp8,0,0.04303466777006785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,64,128,1,fp8,fp8,0,0.054192001620928444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,64,0,1,float16,fp8,0,0.06514133512973785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,64,0,1,fp8,fp8,0,0.0621973325808843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,64,128,1,float16,float16,0,0.029802667597929638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,64,0,1,float16,float16,0,0.05704533557097117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,64,128,1,float16,fp8,0,0.029989334444204967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,64,128,1,fp8,fp8,0,0.03666666646798452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,64,0,1,float16,fp8,0,0.057349334160486855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,64,0,1,fp8,fp8,0,0.0558240016301473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,64,128,1,float16,float16,0,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,64,0,1,float16,float16,0,0.0561706672112147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,64,128,1,float16,fp8,0,0.02976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,64,128,1,fp8,fp8,0,0.036533333361148834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,64,0,1,float16,fp8,0,0.056757330894470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,64,0,1,fp8,fp8,0,0.0555626650651296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,64,128,1,float16,float16,0,0.02850666642189026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,64,0,1,float16,float16,0,0.05583466589450836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,64,0,1,float16,float16,0,0.05566933254400889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,64,128,1,float16,fp8,0,0.028410665690898895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,64,128,1,fp8,fp8,0,0.03462400039037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,64,0,1,float16,fp8,0,0.05566399792830149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,64,0,1,fp8,fp8,0,0.05499733487764994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,64,128,1,float16,float16,0,0.028197333216667175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,64,128,1,float16,fp8,0,0.028431999186674755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,64,128,1,fp8,fp8,0,0.03488533447186152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,64,0,1,float16,fp8,0,0.05593599875768026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,64,0,1,fp8,fp8,0,0.05497066676616669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,64,128,1,float16,float16,0,0.17323199907938638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,64,0,1,float16,float16,0,0.2710453271865845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,64,128,1,float16,fp8,0,0.168778657913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,64,128,1,fp8,fp8,0,0.2230986754099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,64,0,1,float16,fp8,0,0.2636373241742452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,64,128,1,float16,float16,0,0.10226133465766907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,64,0,1,fp8,fp8,0,0.25748799244562787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,64,0,1,float16,float16,0,0.14468266566594443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,64,128,1,float16,fp8,0,0.10077333450317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,64,128,1,fp8,fp8,0,0.137445330619812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,64,0,1,float16,fp8,0,0.1443946659564972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,64,0,1,fp8,fp8,0,0.14573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,64,128,1,float16,float16,0,0.0957493285338084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,64,0,1,float16,float16,0,0.14028267065684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,64,128,1,float16,fp8,0,0.09346133470535278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,64,128,1,fp8,fp8,0,0.12569600343704224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,64,0,1,float16,fp8,0,0.13796266913414001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,64,0,1,fp8,fp8,0,0.13647466897964478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,64,128,1,float16,float16,0,0.05562133093674978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,64,0,1,float16,float16,0,0.07999999821186066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,64,128,1,float16,fp8,0,0.055642664432525635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,64,128,1,fp8,fp8,0,0.07725866635640462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,64,0,1,float16,fp8,0,0.08042133351167043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,64,0,1,fp8,fp8,0,0.07979733248551686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,64,128,1,float16,float16,0,0.05194133520126343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,64,0,1,fp8,fp8,0,0.07529599964618683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,64,0,1,float16,float16,0,0.07690666615962982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,64,128,1,float16,fp8,0,0.053029333551724754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,64,128,1,fp8,fp8,0,0.07006933291753133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,64,0,1,float16,fp8,0,0.0775679995616277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,64,128,1,float16,float16,0,0.03987200061480204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,64,0,1,float16,float16,0,0.05034666756788889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,64,128,1,float16,fp8,0,0.03938666731119156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,64,128,1,fp8,fp8,0,0.04385066529115041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,64,0,1,float16,fp8,0,0.05083199838797251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,64,128,1,fp8,fp8,0,0.04316799839337667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,64,0,1,fp8,fp8,0,0.049679999550183616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,64,128,1,float16,float16,0,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,64,0,1,float16,float16,0,0.04975466430187225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,64,128,1,float16,fp8,0,0.040720000863075256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,64,0,1,float16,fp8,0,0.04959466556708018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,64,0,1,fp8,fp8,0,0.04866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,64,128,1,float16,float16,0,0.028064000109831493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,64,0,1,float16,float16,0,0.047024001677831016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,64,128,1,float16,fp8,0,0.027978666126728058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,64,128,1,fp8,fp8,0,0.03509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,64,0,1,float16,fp8,0,0.04670399924119314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,64,0,1,fp8,fp8,0,0.04607999821503957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,64,128,1,float16,float16,0,0.027978666126728058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,64,0,1,fp8,fp8,0,0.04574933151404063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,64,0,1,float16,float16,0,0.04656533400217692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,64,128,1,float16,fp8,0,0.02808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,64,128,1,fp8,fp8,0,0.03463999927043915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,64,0,1,float16,fp8,0,0.04693333307902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,64,128,1,float16,float16,0,0.02651199946800868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,64,0,1,float16,float16,0,0.04571733375390371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,64,128,1,float16,fp8,0,0.026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,64,128,1,fp8,fp8,0,0.033344000577926636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,64,0,1,fp8,fp8,0,0.044778664906819664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,64,0,1,float16,fp8,0,0.046442667643229164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,64,128,1,float16,float16,0,0.0262773334980011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,64,0,1,float16,float16,0,0.04613866905371348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,64,128,1,float16,fp8,0,0.02666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,64,128,1,fp8,fp8,0,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,64,0,1,float16,fp8,0,0.04587199787298838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,64,0,1,fp8,fp8,0,0.044778664906819664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,64,128,1,float16,float16,0,0.20477867126464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,64,0,1,float16,float16,0,0.28702932596206665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,64,0,1,float16,float16,0,0.15054399768511453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,64,128,1,float16,fp8,0,0.20099733273188272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,64,128,1,fp8,fp8,0,0.2667306661605835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,64,0,1,float16,fp8,0,0.282640000184377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,64,0,1,fp8,fp8,0,0.2592586676279704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,64,128,1,float16,float16,0,0.11925333738327026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,64,128,1,float16,fp8,0,0.1178986628850301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,64,128,1,fp8,fp8,0,0.16132266322771707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,64,0,1,float16,fp8,0,0.1511360009511312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,64,0,1,fp8,fp8,0,0.14619200428326926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,64,128,1,float16,float16,0,0.11178132891654968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,64,0,1,float16,float16,0,0.14130666851997375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,64,128,1,float16,fp8,0,0.11100266377131145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,64,128,1,fp8,fp8,0,0.15335999925931296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,64,0,1,float16,fp8,0,0.1411786675453186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,64,0,1,fp8,fp8,0,0.14014400045077005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,64,128,1,float16,float16,0,0.07020266850789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,64,0,1,float16,float16,0,0.08244266609350841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,64,128,1,float16,fp8,0,0.0708000014225642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,64,128,1,fp8,fp8,0,0.0976693332195282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,64,0,1,float16,fp8,0,0.08383466800053914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,64,0,1,fp8,fp8,0,0.08422933022181193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,64,128,1,float16,float16,0,0.06562666594982147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,64,0,1,float16,float16,0,0.07926933467388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,64,128,1,float16,fp8,0,0.06597866614659627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,64,128,1,fp8,fp8,0,0.08562666177749634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,64,0,1,float16,fp8,0,0.07973333199818929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,64,0,1,fp8,fp8,0,0.07832533121109009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,64,128,1,float16,float16,0,0.04265599946180979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,64,0,1,float16,float16,0,0.04925866425037384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,64,128,1,float16,fp8,0,0.04383466641108195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,64,128,1,fp8,fp8,0,0.055306668082873024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,64,0,1,float16,fp8,0,0.050250664353370667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,64,0,1,fp8,fp8,0,0.04795200129350027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,64,128,1,float16,float16,0,0.04264000058174133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,64,0,1,fp8,fp8,0,0.04692799846331278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,64,0,1,float16,float16,0,0.04833066463470459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,64,128,1,float16,fp8,0,0.04271466533342997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,64,128,1,fp8,fp8,0,0.05386666456858317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,64,0,1,float16,fp8,0,0.04829333225886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,64,128,1,float16,float16,0,0.028421332438786823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,64,0,1,float16,float16,0,0.03829866647720337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,64,128,1,float16,fp8,0,0.02886933336655299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,64,128,1,fp8,fp8,0,0.03572266548871994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,64,0,1,float16,fp8,0,0.0386559988061587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,64,0,1,fp8,fp8,0,0.03822933385769526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,64,128,1,float16,float16,0,0.028218666712443035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,64,0,1,float16,float16,0,0.03807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,64,128,1,float16,fp8,0,0.02847466617822647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,64,128,1,fp8,fp8,0,0.034901333351929985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,64,0,1,float16,fp8,0,0.03792533278465271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,64,0,1,fp8,fp8,0,0.03757333258787791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,64,128,1,float16,float16,0,0.026757332185904186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,64,0,1,float16,float16,0,0.036720000207424164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,64,128,1,float16,fp8,0,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,64,128,1,fp8,fp8,0,0.0331839993596077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,64,128,1,float16,fp8,0,0.026399999856948853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,64,0,1,float16,fp8,0,0.03670933345953623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,64,0,1,float16,fp8,0,0.03710933278004328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,64,0,1,fp8,fp8,0,0.03562666724125544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,64,128,1,float16,float16,0,0.02643733223279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,64,0,1,float16,float16,0,0.036533333361148834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,64,128,1,fp8,fp8,0,0.032773333291212715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,64,0,1,fp8,fp8,0,0.035690667728583016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,64,128,1,float16,float16,0,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,64,0,1,fp8,fp8,0,0.03485333422819773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,64,0,1,float16,float16,0,0.036320000886917114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,64,128,1,float16,fp8,0,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,64,128,1,fp8,fp8,0,0.032042667269706726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,64,128,1,fp8,fp8,0,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,64,0,1,float16,fp8,0,0.03632533301909765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,64,128,1,float16,float16,0,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,64,0,1,float16,float16,0,0.03626666714747747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,64,128,1,float16,fp8,0,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,64,0,1,float16,fp8,0,0.03603200117746989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,64,0,1,float16,fp8,0,0.18998400370279947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,64,0,1,fp8,fp8,0,0.03475733349720637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,64,128,1,float16,float16,0,0.16915200153986612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,64,0,1,float16,float16,0,0.19479467471440634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,64,128,1,float16,fp8,0,0.09823466340700786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,64,128,1,fp8,fp8,0,0.1336426635583242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,64,128,1,float16,fp8,0,0.16596800088882446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,64,128,1,fp8,fp8,0,0.22114133834838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,64,0,1,fp8,fp8,0,0.1872160037358602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,64,0,1,float16,float16,0,0.10942400495211284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,64,128,1,float16,float16,0,0.09959466258684795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,64,0,1,float16,fp8,0,0.10756799578666687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,64,0,1,fp8,fp8,0,0.10988799730936687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,64,128,1,float16,float16,0,0.0919040044148763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,64,0,1,float16,float16,0,0.10256000359853108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,64,128,1,float16,fp8,0,0.09148266911506653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,64,128,1,fp8,fp8,0,0.12531200051307678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,64,0,1,float16,fp8,0,0.10140267014503479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,64,0,1,fp8,fp8,0,0.10179733236630757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,64,128,1,float16,float16,0,0.05683733522891998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,64,0,1,float16,float16,0,0.059936001896858215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,64,128,1,float16,fp8,0,0.055455997586250305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,64,128,1,fp8,fp8,0,0.08066133161385854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,64,0,1,float16,fp8,0,0.061535999178886414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,64,0,1,fp8,fp8,0,0.061664000153541565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,64,0,1,float16,fp8,0,0.0590826670328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,64,128,1,float16,float16,0,0.052255998055140175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,64,0,1,float16,float16,0,0.05831466615200043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,64,128,1,float16,fp8,0,0.05282133320967356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,64,128,1,fp8,fp8,0,0.07072000205516815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,64,0,1,fp8,fp8,0,0.057589332262674965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,64,128,1,float16,float16,0,0.03995199998219808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,64,0,1,float16,float16,0,0.03885333240032196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,64,128,1,float16,fp8,0,0.03868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,64,128,1,fp8,fp8,0,0.04363200068473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,64,0,1,float16,fp8,0,0.0390133336186409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,64,0,1,float16,fp8,0,0.03846399982770284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,64,0,1,fp8,fp8,0,0.037477334340413414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,64,128,1,float16,float16,0,0.03661333272854487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,64,128,1,float16,float16,0,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,64,0,1,float16,float16,0,0.038389332592487335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,64,128,1,float16,fp8,0,0.03890133400758108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,64,128,1,fp8,fp8,0,0.042709335684776306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,64,0,1,fp8,fp8,0,0.03633599976698557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,64,0,1,float16,float16,0,0.03384000062942505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,64,128,1,float16,fp8,0,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,64,128,1,fp8,fp8,0,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,64,0,1,float16,fp8,0,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,64,0,1,fp8,fp8,0,0.03276266654332479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,64,128,1,float16,float16,0,0.02701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,64,128,1,float16,float16,0,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,64,0,1,float16,float16,0,0.03214933226505915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,64,0,1,float16,float16,0,0.03285333265860876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,64,128,1,float16,fp8,0,0.027066667874654133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,64,128,1,fp8,fp8,0,0.03382933388153712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,64,0,1,float16,fp8,0,0.0334346666932106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,64,0,1,fp8,fp8,0,0.03259733319282532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,64,128,1,float16,fp8,0,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,64,128,1,fp8,fp8,0,0.031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,64,0,1,float16,fp8,0,0.03209600100914637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,64,0,1,fp8,fp8,0,0.030533333619435627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,64,128,1,float16,float16,0,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,64,0,1,float16,float16,0,0.03176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,64,128,1,float16,fp8,0,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,64,128,1,fp8,fp8,0,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,64,0,1,float16,fp8,0,0.031471999982992806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,64,0,1,fp8,fp8,0,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,64,128,1,float16,float16,0,0.024480000138282776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,64,0,1,float16,float16,0,0.030613332986831665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,64,128,1,float16,fp8,0,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,64,128,1,fp8,fp8,0,0.031141333281993866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,64,0,1,float16,fp8,0,0.031285333136717476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,64,0,1,fp8,fp8,0,0.030037333567937214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,64,128,1,float16,float16,0,0.024842667082945507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,64,0,1,fp8,fp8,0,0.029445332785447437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,64,0,1,float16,float16,0,0.03092266619205475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,64,128,1,float16,fp8,0,0.024720000723997753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,64,128,1,fp8,fp8,0,0.031061333914597828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,64,0,1,float16,fp8,0,0.031013332307338715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,64,128,1,float16,float16,0,0.18290666739145914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,64,0,1,float16,float16,0,0.2037173310915629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,64,128,1,float16,fp8,0,0.17934399843215942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,64,128,1,fp8,fp8,0,0.24140799045562744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,64,0,1,float16,fp8,0,0.20043732722600302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,64,0,1,fp8,fp8,0,0.1795466740926107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,64,128,1,float16,float16,0,0.11235733826955159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,64,0,1,float16,float16,0,0.1183093289534251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,64,128,1,float16,fp8,0,0.11020800471305847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,64,128,1,fp8,fp8,0,0.14442666371663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,64,0,1,float16,fp8,0,0.11506133278210957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,64,0,1,fp8,fp8,0,0.1053706705570221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,64,128,1,float16,float16,0,0.10454932848612468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,64,0,1,float16,float16,0,0.10834133625030518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,64,128,1,float16,fp8,0,0.10362133383750916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,64,128,1,fp8,fp8,0,0.1394613285859426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,64,0,1,float16,fp8,0,0.10851200421651204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,64,0,1,fp8,fp8,0,0.10076266527175903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,64,128,1,float16,float16,0,0.06493333478768666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,64,0,1,float16,float16,0,0.06533866624037425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,64,128,1,float16,fp8,0,0.0639573335647583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,64,128,1,fp8,fp8,0,0.08747733632723491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,64,0,1,float16,fp8,0,0.06420266628265381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,64,0,1,fp8,fp8,0,0.06294399996598561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,64,128,1,float16,float16,0,0.06018133461475372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,64,128,1,float16,fp8,0,0.05947199960549673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,64,0,1,float16,float16,0,0.061792001128196716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,64,128,1,fp8,fp8,0,0.07745600243409474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,64,128,1,float16,fp8,0,0.04092800120512644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,64,0,1,float16,fp8,0,0.062074666221936546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,64,0,1,fp8,fp8,0,0.057717333237330117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,64,128,1,float16,float16,0,0.04124800115823746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,64,0,1,float16,float16,0,0.03961066653331121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,64,128,1,fp8,fp8,0,0.05141866703828176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,64,0,1,float16,fp8,0,0.039333333571751915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,64,0,1,fp8,fp8,0,0.03603200117746989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,64,0,1,fp8,fp8,0,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,64,128,1,float16,float16,0,0.039861333866914116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,64,0,1,float16,float16,0,0.03875733415285746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,64,128,1,float16,fp8,0,0.027285332481066387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,64,128,1,float16,fp8,0,0.0401706670721372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,64,0,1,float16,fp8,0,0.029861333469549816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,64,128,1,fp8,fp8,0,0.05093866586685181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,64,0,1,float16,fp8,0,0.03882133215665817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,64,128,1,float16,float16,0,0.026943999032179516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,64,0,1,float16,float16,0,0.02957333376010259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,64,128,1,fp8,fp8,0,0.033973333736260734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,64,0,1,fp8,fp8,0,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,64,128,1,float16,float16,0,0.02659733345111211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,64,0,1,float16,float16,0,0.02908266584078471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,64,128,1,float16,fp8,0,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,64,128,1,fp8,fp8,0,0.033488000432650246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,64,0,1,float16,fp8,0,0.028762665887673695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,64,0,1,fp8,fp8,0,0.02773866554101308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,64,128,1,float16,float16,0,0.024117333193620045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,64,0,1,float16,float16,0,0.02678400029738744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,64,128,1,float16,fp8,0,0.024373332659403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,64,128,1,fp8,fp8,0,0.031136001149813335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,64,0,1,float16,fp8,0,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,64,0,1,fp8,fp8,0,0.025989333788553875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,64,128,1,float16,float16,0,0.023989332218964893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,64,0,1,float16,float16,0,0.0269813338915507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,64,128,1,float16,fp8,0,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,64,128,1,fp8,fp8,0,0.031045332551002502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,64,0,1,float16,fp8,0,0.026736001173655193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,64,0,1,fp8,fp8,0,0.02589866767326991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,64,0,1,fp8,fp8,0,0.025263999899228413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,64,128,1,float16,float16,0,0.0239680012067159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,64,128,1,float16,float16,0,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,64,0,1,float16,float16,0,0.02595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,64,128,1,float16,fp8,0,0.023893333971500397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,64,128,1,fp8,fp8,0,0.0305173322558403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,64,0,1,float16,fp8,0,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,64,0,1,float16,float16,0,0.025839999318122864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,64,128,1,float16,fp8,0,0.023936000963052113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,64,128,1,fp8,fp8,0,0.03036266565322876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,64,0,1,float16,fp8,0,0.026074667771657307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,64,0,1,fp8,fp8,0,0.024906667570273083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,64,128,1,float16,float16,0,0.022821334501107533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,64,0,1,float16,float16,0,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,64,128,1,float16,fp8,0,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,64,0,1,float16,float16,0,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,64,128,1,fp8,fp8,0,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,64,0,1,float16,fp8,0,0.02584533393383026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,64,0,1,fp8,fp8,0,0.024533333877722423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,64,128,1,float16,float16,0,0.022858666876951855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,64,128,1,float16,fp8,0,0.023557332654794056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,64,128,1,fp8,fp8,0,0.030053332448005676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,64,0,1,float16,fp8,0,0.025989333788553875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,64,0,1,fp8,fp8,0,0.024175999065240223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,64,128,1,float16,float16,0,0.17391467094421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,64,0,1,float16,float16,0,0.16963199774424234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,64,128,1,float16,fp8,0,0.1728853384653727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,64,128,1,fp8,fp8,0,0.22558399041493735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,64,0,1,float16,fp8,0,0.16718933979670206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,64,128,1,float16,float16,0,0.10875733693440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,64,0,1,fp8,fp8,0,0.15040533741315207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,64,0,1,float16,float16,0,0.10614400108655293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,64,128,1,float16,fp8,0,0.10705600182215373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,64,128,1,fp8,fp8,0,0.1369493305683136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,64,0,1,float16,fp8,0,0.10388799508412679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,64,0,1,fp8,fp8,0,0.09285333752632141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,64,128,1,float16,float16,0,0.09999466935793559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,64,0,1,float16,float16,0,0.092357337474823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,64,128,1,float16,fp8,0,0.10107200344403584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,64,128,1,fp8,fp8,0,0.1318719983100891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,64,0,1,float16,fp8,0,0.09196266531944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,64,128,1,float16,float16,0,0.06227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,64,0,1,fp8,fp8,0,0.08699733018875122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,64,0,1,float16,float16,0,0.05816000203291575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,64,128,1,float16,fp8,0,0.06152533491452535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,64,128,1,fp8,fp8,0,0.08582400282224019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,64,0,1,float16,fp8,0,0.05685866872469584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,64,0,1,fp8,fp8,0,0.05435733497142792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,64,128,1,float16,float16,0,0.057573333382606506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,64,0,1,float16,float16,0,0.053264002005259194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,64,128,1,float16,fp8,0,0.057717333237330117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,64,128,1,fp8,fp8,0,0.07820266485214233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,64,0,1,float16,fp8,0,0.05376533170541128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,64,0,1,fp8,fp8,0,0.04922133187452952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,64,128,1,float16,float16,0,0.04137066751718521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,64,0,1,float16,float16,0,0.03497066597143809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,64,128,1,float16,fp8,0,0.041093334555625916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,64,128,1,fp8,fp8,0,0.05159999926884969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,64,0,1,float16,fp8,0,0.03508266558249792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,64,0,1,fp8,fp8,0,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,64,128,1,float16,float16,0,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,64,0,1,float16,float16,0,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,64,128,1,float16,fp8,0,0.03956799954175949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,64,128,1,fp8,fp8,0,0.05072533090909322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,64,0,1,float16,fp8,0,0.03489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,64,0,1,fp8,fp8,0,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,64,128,1,float16,float16,0,0.027322667340437572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,64,0,1,float16,float16,0,0.02571200082699458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,64,128,1,float16,fp8,0,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,64,128,1,fp8,fp8,0,0.03382933388153712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,64,0,1,float16,fp8,0,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,64,0,1,fp8,fp8,0,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,64,128,1,float16,float16,0,0.026736001173655193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,64,0,1,float16,float16,0,0.024304000039895374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,64,128,1,float16,fp8,0,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,64,128,1,fp8,fp8,0,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,64,0,1,float16,fp8,0,0.025029333929220837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,64,0,1,fp8,fp8,0,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,64,128,1,float16,float16,0,0.02407466620206833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,64,0,1,float16,float16,0,0.022469334304332733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,64,128,1,float16,fp8,0,0.024885334074497223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,64,128,1,fp8,fp8,0,0.030965333183606465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,64,0,1,float16,fp8,0,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,64,0,1,fp8,fp8,0,0.021935999393463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,64,128,1,float16,float16,0,0.02463999887307485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,64,0,1,float16,float16,0,0.02222399910291036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,64,128,1,float16,fp8,0,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,64,128,1,float16,fp8,0,0.024341332415739696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,64,128,1,fp8,fp8,0,0.031184000273545582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,64,0,1,float16,fp8,0,0.02252800017595291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,64,0,1,fp8,fp8,0,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,64,128,1,float16,float16,0,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,64,0,1,float16,float16,0,0.021546666820844013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,64,128,1,fp8,fp8,0,0.030282666285832722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,64,0,1,float16,fp8,0,0.02181866765022278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,64,0,1,fp8,fp8,0,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,64,0,1,fp8,fp8,0,0.020917333662509918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,64,128,1,float16,float16,0,0.023408000667889912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,64,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,64,128,1,float16,fp8,0,0.02390933285156886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,64,128,1,fp8,fp8,0,0.029994666576385498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,64,0,1,float16,fp8,0,0.021781332790851593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,64,128,1,float16,float16,0,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,64,0,1,float16,float16,0,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,64,128,1,float16,fp8,0,0.023024000227451324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,64,128,1,fp8,fp8,0,0.029743999242782593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,64,0,1,float16,fp8,0,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,64,0,1,fp8,fp8,0,0.020282667130231857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,64,128,1,float16,float16,0,0.02257599929968516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,64,0,1,float16,float16,0,0.02080533280968666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,64,128,1,float16,fp8,0,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,64,128,1,fp8,fp8,0,0.029781334102153778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,64,0,1,float16,fp8,0,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,64,0,1,fp8,fp8,0,0.01966933285196622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,64,128,1,float16,float16,0,0.02258133391539256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,64,0,1,float16,float16,0,0.020762667059898376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,64,128,1,float16,fp8,0,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,64,128,1,fp8,fp8,0,0.02945599953333537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,64,0,1,float16,fp8,0,0.020714666694402695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,64,0,1,fp8,fp8,0,0.019850666324297588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,64,128,1,float16,float16,0,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,64,0,1,float16,float16,0,0.020784000555674236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,64,128,1,float16,fp8,0,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,64,128,1,fp8,fp8,0,0.028778667251269024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,64,0,1,float16,fp8,0,0.02070933332045873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,64,0,1,fp8,fp8,0,0.020010666300853092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,64,128,1,float16,float16,0,0.08329066634178162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,64,0,1,float16,float16,0,0.08298133313655853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,64,128,1,fp8,fp8,0,0.10442133744557698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,64,128,1,float16,fp8,0,0.08188266555468242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,64,0,1,float16,fp8,0,0.08007466793060303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,64,0,1,fp8,fp8,0,0.07753066718578339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,64,128,1,float16,float16,0,0.0488373339176178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,64,0,1,float16,float16,0,0.0480320006608963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,64,128,1,float16,fp8,0,0.04806933303674062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,64,128,1,fp8,fp8,0,0.06387733419736226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,64,0,1,float16,fp8,0,0.04730666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,64,0,1,fp8,fp8,0,0.04896533489227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,64,128,1,fp8,fp8,0,0.05795733133951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,64,128,1,float16,float16,0,0.04397333165009817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,64,0,1,float16,float16,0,0.04297600189844767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,64,128,1,float16,fp8,0,0.0444160004456838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,64,0,1,float16,fp8,0,0.04301333427429199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,64,0,1,fp8,fp8,0,0.04398933549722036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,64,128,1,float16,float16,0,0.03456533451875051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,64,0,1,float16,float16,0,0.03400533397992452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,64,128,1,float16,fp8,0,0.033999999364217125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,64,128,1,fp8,fp8,0,0.0359199990828832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,64,0,1,float16,fp8,0,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,64,0,1,fp8,fp8,0,0.029071999092896778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,64,128,1,float16,float16,0,0.03252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,64,0,1,float16,float16,0,0.03209066639343897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,64,128,1,float16,fp8,0,0.0328053335348765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,64,128,1,float16,fp8,0,0.02202133337656657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,64,128,1,fp8,fp8,0,0.03543466577927271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,64,0,1,float16,fp8,0,0.032245332996050514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,64,0,1,fp8,fp8,0,0.02863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,64,128,1,float16,float16,0,0.022250667214393616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,64,0,1,float16,float16,0,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,64,128,1,fp8,fp8,0,0.025685332715511322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,64,0,1,float16,fp8,0,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,64,0,1,fp8,fp8,0,0.021738665799299877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,64,128,1,float16,float16,0,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,64,0,1,float16,float16,0,0.020831999679406483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,64,128,1,float16,fp8,0,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,64,128,1,fp8,fp8,0,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,64,0,1,float16,fp8,0,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,64,0,1,fp8,fp8,0,0.020975999534130096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,64,128,1,float16,float16,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,64,0,1,float16,float16,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,64,0,1,float16,float16,0,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,64,128,1,float16,fp8,0,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,64,128,1,fp8,fp8,0,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,64,0,1,float16,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,64,0,1,fp8,fp8,0,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,64,128,1,float16,float16,0,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,64,128,1,float16,fp8,0,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,64,128,1,fp8,fp8,0,0.02253866692384084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,64,0,1,float16,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,64,0,1,fp8,fp8,0,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,64,128,1,float16,float16,0,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,64,0,1,float16,float16,0,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,64,128,1,float16,fp8,0,0.015930666277805965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,64,128,1,fp8,fp8,0,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,64,0,1,float16,fp8,0,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,64,0,1,fp8,fp8,0,0.018645333747069042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,64,128,1,float16,float16,0,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,64,0,1,float16,float16,0,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,64,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,64,128,1,float16,fp8,0,0.0162773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,64,128,1,fp8,fp8,0,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,64,128,1,fp8,fp8,0,0.021781332790851593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,64,0,1,float16,fp8,0,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,64,0,1,float16,fp8,0,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,64,0,1,fp8,fp8,0,0.0185759998857975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,64,128,1,float16,float16,0,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,64,128,1,float16,fp8,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,64,0,1,fp8,fp8,0,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,64,128,1,float16,float16,0,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,64,0,1,float16,float16,0,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,64,128,1,float16,fp8,0,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,64,128,1,fp8,fp8,0,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,64,0,1,float16,fp8,0,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,64,0,1,fp8,fp8,0,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,64,128,1,float16,float16,0,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,64,0,1,float16,float16,0,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,64,128,1,float16,fp8,0,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,64,128,1,fp8,fp8,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,64,0,1,float16,fp8,0,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,64,0,1,float16,fp8,0,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,64,0,1,fp8,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,64,128,1,float16,float16,0,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,64,0,1,float16,float16,0,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,64,128,1,float16,fp8,0,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,64,0,1,fp8,fp8,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,64,128,1,float16,float16,0,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,64,0,1,fp8,fp8,0,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,64,0,1,float16,float16,0,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,64,128,1,float16,fp8,0,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,64,128,1,fp8,fp8,0,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,64,128,1,fp8,fp8,0,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,64,0,1,float16,fp8,0,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,64,128,1,float16,float16,0,0.014618666221698126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,64,0,1,float16,float16,0,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,64,128,1,float16,fp8,0,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,64,128,1,fp8,fp8,0,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,64,0,1,float16,fp8,0,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,64,0,1,fp8,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,64,128,1,float16,float16,0,0.03634133438269297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,64,0,1,float16,float16,0,0.03583466758330663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,64,128,1,float16,fp8,0,0.03606399893760681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,64,128,1,fp8,fp8,0,0.043824002146720886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,64,0,1,float16,fp8,0,0.03606933355331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,64,128,1,float16,float16,0,0.026005332668622334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,64,0,1,fp8,fp8,0,0.04435733457406362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,64,0,1,float16,float16,0,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,64,128,1,float16,fp8,0,0.025765334566434223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,64,128,1,fp8,fp8,0,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,64,0,1,float16,fp8,0,0.02610666553179423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,64,0,1,fp8,fp8,0,0.029109333952267964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,64,128,1,float16,float16,0,0.025797332326571148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,64,0,1,float16,float16,0,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,64,128,1,float16,fp8,0,0.024495999018351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,64,128,1,fp8,fp8,0,0.028512001037597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,64,0,1,float16,fp8,0,0.02446399877468745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,64,0,1,fp8,fp8,0,0.028832000990708668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,64,128,1,float16,float16,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,64,0,1,float16,float16,0,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,64,128,1,float16,fp8,0,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,64,128,1,fp8,fp8,0,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,64,0,1,float16,fp8,0,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,64,0,1,fp8,fp8,0,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,64,128,1,float16,float16,0,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,64,0,1,float16,float16,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,64,128,1,float16,fp8,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,64,128,1,fp8,fp8,0,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,64,0,1,float16,fp8,0,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,64,0,1,fp8,fp8,0,0.02067733307679494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,64,128,1,float16,float16,0,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,64,0,1,float16,float16,0,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,64,128,1,float16,fp8,0,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,64,128,1,fp8,fp8,0,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,64,0,1,float16,fp8,0,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,64,0,1,fp8,fp8,0,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,64,128,1,float16,float16,0,0.01444799949725469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,64,0,1,float16,float16,0,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,64,128,1,float16,fp8,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,64,128,1,fp8,fp8,0,0.019930666933457058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,64,0,1,float16,fp8,0,0.014725333700577417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,64,0,1,fp8,fp8,0,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,64,128,1,float16,float16,0,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,64,0,1,float16,float16,0,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,64,128,1,float16,fp8,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,64,128,1,float16,fp8,0,0.014122666170199713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,64,128,1,fp8,fp8,0,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,64,128,1,fp8,fp8,0,0.0183999997874101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,64,0,1,float16,fp8,0,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,64,0,1,fp8,fp8,0,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,64,128,1,float16,float16,0,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,64,0,1,float16,float16,0,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,64,0,1,float16,fp8,0,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,64,0,1,fp8,fp8,0,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,64,128,1,float16,float16,0,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,64,0,1,float16,float16,0,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,64,128,1,float16,fp8,0,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,64,128,1,fp8,fp8,0,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,64,0,1,float16,fp8,0,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,64,0,1,fp8,fp8,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,64,128,1,float16,float16,0,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,64,0,1,float16,float16,0,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,64,128,1,float16,fp8,0,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,64,128,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,64,0,1,float16,fp8,0,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,64,0,1,fp8,fp8,0,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,64,128,1,float16,float16,0,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,64,0,1,float16,float16,0,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,64,128,1,float16,fp8,0,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,64,128,1,fp8,fp8,0,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,64,0,1,float16,fp8,0,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,64,0,1,float16,fp8,0,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,64,0,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,64,128,1,float16,float16,0,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,64,0,1,float16,float16,0,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,64,128,1,float16,fp8,0,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,64,128,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,64,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,64,128,1,float16,float16,0,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,64,0,1,float16,float16,0,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,64,128,1,float16,fp8,0,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,64,128,1,fp8,fp8,0,0.017535999417304993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,64,0,1,float16,fp8,0,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,64,0,1,fp8,fp8,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,64,128,1,float16,float16,0,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,64,0,1,float16,float16,0,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,64,128,1,float16,fp8,0,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,64,128,1,fp8,fp8,0,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,64,0,1,float16,fp8,0,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,64,0,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,64,128,1,float16,float16,0,0.012383999923865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,64,0,1,float16,float16,0,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,64,128,1,float16,fp8,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,64,128,1,fp8,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,64,0,1,float16,fp8,0,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,64,0,1,fp8,fp8,0,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,64,128,1,float16,float16,0,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,64,0,1,float16,float16,0,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,64,128,1,float16,fp8,0,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,64,128,1,fp8,fp8,0,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,64,0,1,float16,fp8,0,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,64,0,1,float16,float16,0,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,64,0,1,fp8,fp8,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,64,128,1,float16,float16,0,0.022330666581789654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,64,0,1,float16,fp8,0,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,64,128,1,float16,fp8,0,0.022618666291236877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,64,128,1,fp8,fp8,0,0.03465600063403448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,64,128,1,float16,float16,0,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,64,0,1,fp8,fp8,0,0.034287999073664345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,64,0,1,float16,float16,0,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,64,128,1,float16,fp8,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,64,128,1,fp8,fp8,0,0.02465066562096278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,64,0,1,float16,fp8,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,64,0,1,fp8,fp8,0,0.025146665672461193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,64,128,1,float16,float16,0,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,64,0,1,float16,float16,0,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,64,128,1,float16,fp8,0,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,64,128,1,fp8,fp8,0,0.024282666544119518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,64,0,1,float16,fp8,0,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,64,0,1,fp8,fp8,0,0.02447466552257538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,64,0,1,float16,fp8,0,0.014074667046467463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,64,128,1,float16,float16,0,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,64,0,1,float16,float16,0,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,64,128,1,float16,float16,0,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,64,128,1,float16,fp8,0,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,64,128,1,fp8,fp8,0,0.019685332973798115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,64,0,1,fp8,fp8,0,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,64,0,1,fp8,fp8,0,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,64,0,1,float16,float16,0,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,64,128,1,float16,fp8,0,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,64,128,1,fp8,fp8,0,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,64,0,1,float16,fp8,0,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,64,128,1,float16,float16,0,0.012319999436537424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,64,0,1,float16,float16,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,64,128,1,float16,fp8,0,0.012373333175977072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,64,128,1,fp8,fp8,0,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,64,0,1,float16,fp8,0,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,64,0,1,fp8,fp8,0,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,64,128,1,float16,float16,0,0.012245333443085352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,64,0,1,float16,float16,0,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,64,128,1,float16,fp8,0,0.012416000167528788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,64,128,1,fp8,fp8,0,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,64,0,1,float16,fp8,0,0.012495999534924826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,64,0,1,fp8,fp8,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,64,128,1,float16,float16,0,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,64,0,1,float16,float16,0,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,64,128,1,float16,fp8,0,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,64,128,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,64,0,1,float16,fp8,0,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,64,0,1,fp8,fp8,0,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,64,128,1,float16,float16,0,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,64,0,1,float16,float16,0,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,64,128,1,float16,fp8,0,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,64,128,1,fp8,fp8,0,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,64,0,1,float16,fp8,0,0.012074666718641916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,64,0,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,64,128,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,64,0,1,float16,float16,0,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,64,128,1,float16,fp8,0,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,64,128,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,64,0,1,float16,fp8,0,0.012298667182525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,64,0,1,fp8,fp8,0,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,64,0,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,64,128,1,float16,float16,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,64,128,1,float16,float16,0,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,64,0,1,float16,float16,0,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,64,128,1,float16,fp8,0,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,64,128,1,fp8,fp8,0,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,64,0,1,float16,fp8,0,0.01202133297920227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,64,0,1,float16,float16,0,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,64,128,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,64,128,1,fp8,fp8,0,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,64,0,1,float16,fp8,0,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,64,0,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,64,0,1,fp8,fp8,0,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,64,128,1,float16,float16,0,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,64,0,1,float16,float16,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,64,128,1,float16,fp8,0,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,64,128,1,fp8,fp8,0,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,64,0,1,float16,fp8,0,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,64,128,1,float16,float16,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,64,0,1,float16,float16,0,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,64,128,1,float16,fp8,0,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,64,128,1,fp8,fp8,0,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,64,0,1,float16,fp8,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,64,0,1,fp8,fp8,0,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,64,128,1,float16,float16,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,64,0,1,float16,float16,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,64,128,1,float16,fp8,0,0.01202133297920227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,64,128,1,fp8,fp8,0,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,64,0,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,64,0,1,fp8,fp8,0,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,64,128,1,float16,float16,0,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,64,0,1,float16,float16,0,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,64,128,1,float16,fp8,0,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,64,128,1,fp8,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,64,0,1,float16,fp8,0,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,64,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,64,128,1,float16,float16,0,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,64,0,1,float16,float16,0,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,64,128,1,float16,fp8,0,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,64,128,1,fp8,fp8,0,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,64,0,1,float16,fp8,0,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,64,0,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,64,128,1,float16,float16,0,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,64,0,1,float16,float16,0,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,64,128,1,float16,fp8,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,64,128,1,fp8,fp8,0,0.030666666726271313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,64,0,1,float16,fp8,0,0.016741332908471424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,64,0,1,fp8,fp8,0,0.030960001051425934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,64,128,1,float16,float16,0,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,64,0,1,float16,float16,0,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,64,0,1,fp8,fp8,0,0.022863999009132385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,64,128,1,float16,fp8,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,64,0,1,float16,float16,0,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,64,128,1,fp8,fp8,0,0.022682666778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,64,0,1,float16,fp8,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,64,128,1,float16,float16,0,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,64,128,1,float16,fp8,0,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,64,128,1,fp8,fp8,0,0.022495999932289124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,64,128,1,float16,fp8,0,0.01228800043463707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,64,0,1,float16,fp8,0,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,64,0,1,fp8,fp8,0,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,64,128,1,float16,float16,0,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,64,0,1,float16,float16,0,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,64,128,1,fp8,fp8,0,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,64,0,1,float16,fp8,0,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,64,0,1,fp8,fp8,0,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,64,0,1,float16,fp8,0,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,64,128,1,float16,float16,0,0.011920000116030375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,64,0,1,float16,float16,0,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,64,128,1,float16,fp8,0,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,64,128,1,fp8,fp8,0,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,64,0,1,fp8,fp8,0,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,64,128,1,float16,float16,0,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,64,0,1,float16,float16,0,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,64,128,1,float16,fp8,0,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,64,128,1,fp8,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,64,0,1,float16,fp8,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,64,0,1,fp8,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,64,128,1,float16,float16,0,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,64,0,1,float16,float16,0,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,64,128,1,float16,fp8,0,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,64,128,1,fp8,fp8,0,0.01817600056529045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,64,0,1,float16,fp8,0,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,64,0,1,fp8,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,64,128,1,float16,float16,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,64,0,1,float16,float16,0,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,64,128,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,64,128,1,fp8,fp8,0,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,64,0,1,float16,fp8,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,64,0,1,fp8,fp8,0,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,64,128,1,float16,float16,0,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,64,0,1,float16,float16,0,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,64,128,1,float16,fp8,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,64,128,1,fp8,fp8,0,0.017893332988023758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,64,0,1,float16,fp8,0,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,64,0,1,fp8,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,64,128,1,float16,float16,0,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,64,0,1,float16,float16,0,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,64,128,1,float16,fp8,0,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,64,128,1,fp8,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,64,0,1,float16,fp8,0,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,64,0,1,fp8,fp8,0,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,64,128,1,float16,float16,0,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,64,0,1,float16,float16,0,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,64,128,1,float16,fp8,0,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,64,128,1,fp8,fp8,0,0.01773333301146825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,64,0,1,float16,fp8,0,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,64,0,1,fp8,fp8,0,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,64,128,1,float16,float16,0,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,64,0,1,float16,float16,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,64,128,1,float16,fp8,0,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,64,128,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,64,0,1,float16,fp8,0,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,64,0,1,fp8,fp8,0,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,64,128,1,float16,float16,0,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,64,0,1,float16,float16,0,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,64,128,1,float16,fp8,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,64,128,1,fp8,fp8,0,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,64,128,1,fp8,fp8,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,64,0,1,float16,fp8,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,64,0,1,float16,fp8,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,64,0,1,fp8,fp8,0,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,64,128,1,float16,float16,0,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,64,0,1,float16,float16,0,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,64,128,1,float16,fp8,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,64,0,1,fp8,fp8,0,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,64,128,1,float16,float16,0,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,64,0,1,float16,float16,0,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,64,128,1,float16,fp8,0,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,64,128,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,64,0,1,float16,fp8,0,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,64,0,1,fp8,fp8,0,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,64,128,1,float16,float16,0,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,64,0,1,fp8,fp8,0,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,64,0,1,float16,float16,0,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,64,128,1,float16,fp8,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,64,128,1,fp8,fp8,0,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,64,0,1,float16,fp8,0,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,64,128,1,float16,float16,0,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,64,0,1,float16,float16,0,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,64,128,1,float16,fp8,0,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,64,128,1,fp8,fp8,0,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,64,0,1,float16,float16,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,64,0,1,float16,fp8,0,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,64,128,1,float16,fp8,0,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,64,0,1,fp8,fp8,0,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,64,128,1,float16,float16,0,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,64,0,1,float16,float16,0,0.012421333541472753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,64,128,1,fp8,fp8,0,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,64,128,1,float16,fp8,0,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,64,0,1,float16,fp8,0,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,64,0,1,fp8,fp8,0,0.02886933336655299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,64,128,1,float16,float16,0,0.012565333396196365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,64,128,1,fp8,fp8,0,0.021744000415007275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,64,0,1,float16,fp8,0,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,64,0,1,fp8,fp8,0,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,64,128,1,float16,float16,0,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,64,0,1,fp8,fp8,0,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,64,0,1,float16,float16,0,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,64,128,1,float16,fp8,0,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,64,128,1,fp8,fp8,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,64,128,1,fp8,fp8,0,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,64,0,1,float16,fp8,0,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,64,128,1,float16,float16,0,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,64,0,1,float16,float16,0,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,64,128,1,float16,fp8,0,0.01202133297920227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,64,0,1,float16,fp8,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,64,0,1,fp8,fp8,0,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,64,0,1,float16,fp8,0,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,64,128,1,float16,float16,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,64,0,1,float16,float16,0,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,64,128,1,float16,fp8,0,0.012346666306257248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,64,128,1,fp8,fp8,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,64,0,1,fp8,fp8,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,64,128,1,float16,float16,0,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,64,0,1,float16,float16,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,64,128,1,float16,fp8,0,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,64,128,1,fp8,fp8,0,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,64,0,1,float16,fp8,0,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,64,0,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,64,128,1,float16,float16,0,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,64,0,1,float16,float16,0,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,64,128,1,float16,fp8,0,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,64,128,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,64,0,1,float16,fp8,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,64,0,1,fp8,fp8,0,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,64,128,1,float16,float16,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,64,0,1,float16,float16,0,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,64,128,1,float16,fp8,0,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,64,128,1,fp8,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,64,0,1,float16,fp8,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,64,0,1,float16,fp8,0,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,64,0,1,fp8,fp8,0,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,64,128,1,float16,float16,0,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,64,0,1,float16,float16,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,64,128,1,float16,fp8,0,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,64,128,1,fp8,fp8,0,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,64,0,1,fp8,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,64,128,1,float16,float16,0,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,64,0,1,float16,float16,0,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,64,128,1,float16,fp8,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,64,128,1,fp8,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,64,0,1,float16,fp8,0,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,64,0,1,float16,fp8,0,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,64,0,1,fp8,fp8,0,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,64,128,1,float16,float16,0,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,64,0,1,float16,float16,0,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,64,128,1,float16,fp8,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,64,128,1,fp8,fp8,0,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,64,0,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,64,128,1,float16,float16,0,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,64,0,1,float16,float16,0,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,64,128,1,float16,fp8,0,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,64,128,1,fp8,fp8,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,64,0,1,float16,fp8,0,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,64,0,1,fp8,fp8,0,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,64,128,1,float16,float16,0,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,64,0,1,float16,float16,0,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,64,128,1,float16,fp8,0,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,64,128,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,64,0,1,float16,fp8,0,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,64,0,1,float16,fp8,0,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,64,0,1,fp8,fp8,0,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,64,128,1,float16,float16,0,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,64,0,1,float16,float16,0,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,64,128,1,float16,fp8,0,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,64,128,1,float16,fp8,0,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,64,128,1,fp8,fp8,0,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,64,0,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,64,128,1,float16,float16,0,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,64,0,1,float16,float16,0,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,64,128,1,fp8,fp8,0,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,64,0,1,float16,fp8,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,64,0,1,fp8,fp8,0,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,64,128,1,float16,float16,0,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,64,0,1,float16,float16,0,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,64,128,1,float16,fp8,0,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,64,128,1,fp8,fp8,0,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,64,0,1,float16,fp8,0,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,64,0,1,fp8,fp8,0,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,64,128,1,float16,float16,0,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,64,0,1,float16,float16,0,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,64,128,1,float16,fp8,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,64,128,1,fp8,fp8,0,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,64,0,1,float16,fp8,0,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,64,0,1,fp8,fp8,0,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,64,128,1,float16,float16,0,0.07365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,64,128,1,float16,fp8,0,0.07373333474000295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,64,128,1,fp8,fp8,0,0.10058666268984477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,64,0,1,float16,float16,0,0.3168800075848897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,64,128,1,float16,float16,0,0.046480000019073486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,64,0,1,float16,fp8,0,0.31782933076222736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,64,0,1,fp8,fp8,0,0.29739199082056683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,64,128,1,float16,fp8,0,0.04710400104522705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,64,0,1,float16,float16,0,0.1665493349234263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,64,128,1,fp8,fp8,0,0.057317331433296204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,64,0,1,float16,fp8,0,0.1675306757291158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,64,0,1,fp8,fp8,0,0.1575146714846293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,64,128,1,float16,float16,0,0.035536001125971474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,64,128,1,float16,fp8,0,0.035589332381884255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,64,0,1,float16,float16,0,0.16478932897249857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,64,128,1,fp8,fp8,0,0.04155733436346054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,64,0,1,fp8,fp8,0,0.15507733821868896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,64,0,1,float16,fp8,0,0.16380266348520914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,64,128,1,float16,float16,0,0.05913066864013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,64,128,1,float16,fp8,0,0.06062933305899302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,64,0,1,float16,float16,0,0.2050079902013143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,64,128,1,fp8,fp8,0,0.07981866598129272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,64,0,1,fp8,fp8,0,0.19134400288263956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,64,0,1,float16,fp8,0,0.20505066712697348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,64,0,1,float16,float16,0,0.12825600306193033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,64,128,1,float16,float16,0,0.0425546665986379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,64,128,1,float16,fp8,0,0.04287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,64,128,1,fp8,fp8,0,0.0446720023949941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,64,0,1,float16,fp8,0,0.1295146644115448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,64,0,1,fp8,fp8,0,0.12214932839075725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,64,128,1,fp8,fp8,0,0.03865066667397817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,64,0,1,float16,fp8,0,0.1264586647351583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,64,128,1,float16,float16,0,0.032272001107533775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,64,0,1,float16,float16,0,0.12762133280436197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,64,128,1,float16,fp8,0,0.03224000086386999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,64,0,1,fp8,fp8,0,0.12049600481987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,64,128,1,float16,float16,0,0.05036266644795736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,64,0,1,float16,float16,0,0.15221866965293884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,64,128,1,fp8,fp8,0,0.0626453310251236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,64,128,1,float16,fp8,0,0.051781331499417625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,64,0,1,float16,fp8,0,0.14919466773668924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,64,128,1,float16,float16,0,0.039290666580200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,64,0,1,fp8,fp8,0,0.1402079959710439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,64,0,1,float16,float16,0,0.10997333129247029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,64,128,1,float16,fp8,0,0.03722133239110311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,64,128,1,fp8,fp8,0,0.042453333735466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,64,0,1,float16,fp8,0,0.10946666200955708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,64,0,1,fp8,fp8,0,0.10471466183662415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,64,128,1,float16,float16,0,0.03012266755104065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,64,0,1,float16,float16,0,0.10807466506958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,64,128,1,float16,fp8,0,0.03051200012365977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,64,128,1,fp8,fp8,0,0.036789332826932274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,64,0,1,float16,fp8,0,0.10859733819961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,64,0,1,fp8,fp8,0,0.10374400019645691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,64,128,1,float16,float16,0,0.07179200152556102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,64,0,1,float16,float16,0,0.1848106582959493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,64,128,1,float16,fp8,0,0.072202667593956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,64,128,1,fp8,fp8,0,0.0995093286037445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,64,0,1,float16,fp8,0,0.18519467115402222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,64,0,1,fp8,fp8,0,0.17629333337148032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,64,128,1,float16,float16,0,0.04314666489760081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,64,0,1,float16,fp8,0,0.09880000352859497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,64,0,1,float16,float16,0,0.09794132908185323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,64,128,1,float16,fp8,0,0.04496533175309499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,64,128,1,fp8,fp8,0,0.056554665168126426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,64,128,1,float16,float16,0,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,64,0,1,fp8,fp8,0,0.09417600433031718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,64,0,1,float16,float16,0,0.09147199988365173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,64,128,1,float16,fp8,0,0.030634666482607525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,64,128,1,fp8,fp8,0,0.03658133248488108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,64,0,1,float16,fp8,0,0.09172266721725464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,64,0,1,fp8,fp8,0,0.08825066685676575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,64,128,1,float16,float16,0,0.029135999580224354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,64,0,1,float16,float16,0,0.0900320013364156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,64,128,1,float16,fp8,0,0.028618666032950085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,64,128,1,fp8,fp8,0,0.03555200000603994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,64,0,1,float16,fp8,0,0.0905013382434845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,64,0,1,fp8,fp8,0,0.08789867162704468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,64,128,1,float16,float16,0,0.057130664587020874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,64,128,1,float16,fp8,0,0.05575466652711233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,64,0,1,float16,float16,0,0.122789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,64,128,1,fp8,fp8,0,0.08116266628106435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,64,0,1,float16,fp8,0,0.12069333593050639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,64,0,1,fp8,fp8,0,0.11756799618403117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,64,128,1,float16,float16,0,0.04083200047413508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,64,0,1,float16,float16,0,0.0758133331934611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,64,128,1,float16,fp8,0,0.041349334021409355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,64,128,1,fp8,fp8,0,0.04366933306058248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,64,0,1,float16,fp8,0,0.07627200086911519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,64,0,1,fp8,fp8,0,0.07441600163777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,64,128,1,float16,float16,0,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,64,0,1,float16,float16,0,0.07251733541488647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,64,128,1,float16,fp8,0,0.028757333755493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,64,128,1,fp8,fp8,0,0.03462400039037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,64,0,1,float16,fp8,0,0.0727040022611618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,64,0,1,fp8,fp8,0,0.07207466661930084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,64,128,1,float16,float16,0,0.02698666602373123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,64,0,1,float16,float16,0,0.07140799860159557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,64,128,1,float16,fp8,0,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,64,128,1,fp8,fp8,0,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,64,0,1,float16,fp8,0,0.07161066432793935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,64,0,1,fp8,fp8,0,0.06975999971230824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,64,128,1,float16,float16,0,0.0718560020128886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,64,0,1,float16,float16,0,0.11777066191037495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,64,128,1,float16,fp8,0,0.07247466842333476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,64,128,1,fp8,fp8,0,0.0986293355623881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,64,0,1,float16,fp8,0,0.11899200081825256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,64,0,1,fp8,fp8,0,0.11460266510645549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,64,128,1,float16,float16,0,0.04312533140182495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,64,0,1,float16,float16,0,0.06588266789913177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,64,128,1,float16,fp8,0,0.043605332573254905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,64,128,1,fp8,fp8,0,0.05516799787680308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,64,0,1,float16,fp8,0,0.06678933401902516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,64,0,1,fp8,fp8,0,0.06397333244482677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,64,128,1,float16,float16,0,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,64,0,1,float16,float16,0,0.05514133473237356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,64,128,1,float16,fp8,0,0.029135999580224354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,64,128,1,fp8,fp8,0,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,64,0,1,float16,fp8,0,0.055530667304992676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,64,0,1,fp8,fp8,0,0.055045331517855324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,64,128,1,float16,float16,0,0.026496000587940216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,64,0,1,float16,float16,0,0.054085334142049156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,64,128,1,float16,fp8,0,0.026330667237440746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,64,128,1,fp8,fp8,0,0.032986665765444435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,64,0,1,float16,fp8,0,0.053632001082102455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,64,128,1,fp8,fp8,0,0.03179733455181122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,64,0,1,fp8,fp8,0,0.05273066461086273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,64,128,1,float16,float16,0,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,64,0,1,float16,float16,0,0.052970667680104576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,64,128,1,float16,fp8,0,0.026261332134405773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,64,0,1,float16,fp8,0,0.05356266597906748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,64,0,1,fp8,fp8,0,0.05150933563709259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,64,128,1,float16,float16,0,0.056202664971351624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,64,0,1,float16,float16,0,0.08027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,64,128,1,float16,fp8,0,0.055173332492510475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,64,128,1,fp8,fp8,0,0.07758933305740356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,64,0,1,float16,fp8,0,0.08135466774304707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,64,0,1,fp8,fp8,0,0.07895466685295105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,64,128,1,float16,float16,0,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,64,128,1,float16,float16,0,0.027690666417280834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,64,0,1,float16,float16,0,0.04983466863632202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,64,128,1,fp8,fp8,0,0.043568000197410583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,64,128,1,float16,fp8,0,0.03745066622893015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,64,0,1,float16,fp8,0,0.05007466673851013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,64,0,1,fp8,fp8,0,0.04995200037956238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,64,0,1,float16,float16,0,0.0458186666170756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,64,128,1,float16,fp8,0,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,64,128,1,fp8,fp8,0,0.03379733363787333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,64,0,1,float16,fp8,0,0.04586133360862732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,64,0,1,fp8,fp8,0,0.045221333702405296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,64,128,1,float16,float16,0,0.025663999219735462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,64,128,1,float16,float16,0,0.024527999262015026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,64,0,1,float16,float16,0,0.04403733213742574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,64,128,1,float16,fp8,0,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,64,128,1,fp8,fp8,0,0.03161066770553589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,64,0,1,float16,fp8,0,0.04455466568470001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,64,0,1,fp8,fp8,0,0.043375998735427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,64,0,1,float16,float16,0,0.04390400151411692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,64,128,1,float16,fp8,0,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,64,128,1,fp8,fp8,0,0.030837332208951313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,64,0,1,float16,fp8,0,0.04417066772778829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,64,0,1,fp8,fp8,0,0.04274133344491323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,64,128,1,float16,float16,0,0.0722453345855077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,64,0,1,float16,float16,0,0.08523733417193095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,64,128,1,float16,fp8,0,0.07142400244871776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,64,128,1,fp8,fp8,0,0.10044266780217488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,64,0,1,float16,fp8,0,0.0860053300857544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,64,0,1,fp8,fp8,0,0.08496000369389851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,64,128,1,float16,float16,0,0.043968002001444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,64,0,1,float16,float16,0,0.05027733246485392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,64,128,1,float16,fp8,0,0.0441599984963735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,64,128,1,fp8,fp8,0,0.055173332492510475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,64,0,1,float16,fp8,0,0.04986133178075155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,64,0,1,fp8,fp8,0,0.048122664292653404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,64,128,1,float16,float16,0,0.02845333268245061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,64,0,1,float16,float16,0,0.03839466720819473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,64,128,1,float16,fp8,0,0.02849599967400233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,64,128,1,fp8,fp8,0,0.03550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,64,0,1,float16,fp8,0,0.03849600007136663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,64,0,1,fp8,fp8,0,0.03813866774241129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,64,128,1,float16,float16,0,0.025770666698614757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,64,0,1,float16,float16,0,0.0364479993780454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,64,128,1,float16,fp8,0,0.026213333010673523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,64,128,1,fp8,fp8,0,0.0322773332397143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,64,0,1,float16,fp8,0,0.03678400069475174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,64,128,1,fp8,fp8,0,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,64,0,1,fp8,fp8,0,0.036042665441830955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,64,128,1,float16,float16,0,0.02455466737349828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,64,0,1,float16,float16,0,0.0354666660229365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,64,128,1,float16,fp8,0,0.024901332954565685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,64,0,1,float16,fp8,0,0.035616000493367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,64,0,1,fp8,fp8,0,0.034101332227389015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,64,128,1,float16,float16,0,0.024442667762438457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,64,0,1,float16,float16,0,0.03505066782236099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,64,128,1,float16,fp8,0,0.023989332218964893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,64,128,1,fp8,fp8,0,0.030597334106763203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,64,0,1,float16,fp8,0,0.03488533447186152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,64,0,1,fp8,fp8,0,0.03330666571855545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,64,128,1,float16,float16,0,0.05751466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,64,0,1,float16,float16,0,0.062218666076660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,64,128,1,float16,fp8,0,0.05705599983533224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,64,128,1,fp8,fp8,0,0.07691200077533722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,64,0,1,float16,fp8,0,0.06401066482067108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,64,0,1,fp8,fp8,0,0.06372266511122386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,64,128,1,float16,float16,0,0.03997866561015447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,64,0,1,float16,float16,0,0.03872533390919367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,64,128,1,float16,fp8,0,0.03915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,64,128,1,fp8,fp8,0,0.04417066772778829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,64,0,1,float16,fp8,0,0.03996799886226654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,64,0,1,fp8,fp8,0,0.03798400113979975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,64,128,1,float16,float16,0,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,64,0,1,float16,float16,0,0.03325333446264267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,64,0,1,float16,float16,0,0.03196266790231069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,64,128,1,float16,fp8,0,0.027045334378878277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,64,128,1,fp8,fp8,0,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,64,0,1,float16,fp8,0,0.03357866654793421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,64,0,1,fp8,fp8,0,0.03211733450492223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,64,0,1,fp8,fp8,0,0.030991998811562855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,64,128,1,float16,float16,0,0.02565866708755493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,64,128,1,float16,fp8,0,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,64,128,1,fp8,fp8,0,0.0315733328461647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,64,0,1,float16,fp8,0,0.03182400017976761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,64,128,1,float16,float16,0,0.024277334411938984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,64,0,1,float16,float16,0,0.03062933435042699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,64,128,1,float16,fp8,0,0.024282666544119518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,64,128,1,fp8,fp8,0,0.03091199944416682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,64,0,1,float16,fp8,0,0.03091199944416682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,64,0,1,fp8,fp8,0,0.02942399928967158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,64,128,1,float16,float16,0,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,64,0,1,float16,float16,0,0.029696000119050343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,64,128,1,float16,fp8,0,0.0236160010099411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,64,128,1,fp8,fp8,0,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,64,0,1,float16,fp8,0,0.029850666721661884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,64,0,1,fp8,fp8,0,0.028410665690898895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,64,128,1,float16,float16,0,0.06332799792289734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,64,0,1,float16,float16,0,0.06525333225727081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,64,128,1,float16,fp8,0,0.06258666515350342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,64,128,1,fp8,fp8,0,0.08619200189908345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,64,0,1,float16,fp8,0,0.0647573322057724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,64,0,1,fp8,fp8,0,0.06135466694831848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,64,128,1,float16,float16,0,0.041477332512537636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,64,128,1,float16,float16,0,0.026922665536403656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,64,0,1,float16,float16,0,0.0394400010506312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,64,128,1,float16,fp8,0,0.04154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,64,128,1,fp8,fp8,0,0.050437331199645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,64,0,1,float16,fp8,0,0.03972800076007843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,64,0,1,fp8,fp8,0,0.03605866680542628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,64,0,1,float16,float16,0,0.029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,64,128,1,float16,fp8,0,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,64,128,1,fp8,fp8,0,0.033344000577926636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,64,0,1,float16,fp8,0,0.029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,64,0,1,float16,fp8,0,0.027690666417280834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,64,0,1,fp8,fp8,0,0.028255999088287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,64,128,1,float16,float16,0,0.02438933402299881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,64,0,1,float16,float16,0,0.026880001028378803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,64,128,1,float16,fp8,0,0.024688000480333965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,64,128,1,fp8,fp8,0,0.030970667799313862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,64,0,1,fp8,fp8,0,0.026320000489552815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,64,128,1,float16,float16,0,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,64,0,1,float16,float16,0,0.026026666164398193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,64,128,1,float16,fp8,0,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,64,128,1,fp8,fp8,0,0.030026666820049286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,64,128,1,fp8,fp8,0,0.02979733298222224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,64,0,1,float16,fp8,0,0.0262773334980011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,64,0,1,fp8,fp8,0,0.024879999458789825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,64,128,1,float16,float16,0,0.022986667851607006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,64,0,1,float16,float16,0,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,64,128,1,float16,fp8,0,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,64,0,1,float16,fp8,0,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,64,0,1,fp8,fp8,0,0.024527999262015026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,64,128,1,float16,float16,0,0.02242133269707362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,64,0,1,float16,float16,0,0.024890666206677754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,64,128,1,float16,fp8,0,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,64,0,1,float16,float16,0,0.057674666245778404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,64,128,1,fp8,fp8,0,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,64,0,1,float16,fp8,0,0.02601066728432973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,64,0,1,fp8,fp8,0,0.02454400062561035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,64,128,1,float16,float16,0,0.06149866680304209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,64,128,1,float16,fp8,0,0.06050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,64,128,1,fp8,fp8,0,0.08509332935015361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,64,0,1,float16,fp8,0,0.05695466697216034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,64,0,1,fp8,fp8,0,0.05351466437180837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,64,128,1,fp8,fp8,0,0.04978133241335551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,64,128,1,float16,float16,0,0.04018666595220566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,64,0,1,float16,float16,0,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,64,128,1,float16,fp8,0,0.04066666712363561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,64,0,1,float16,fp8,0,0.03506666670242945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,64,0,1,fp8,fp8,0,0.031328000128269196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,64,128,1,float16,float16,0,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,64,0,1,float16,float16,0,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,64,128,1,float16,fp8,0,0.02828266719977061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,64,128,1,fp8,fp8,0,0.03355200091997782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,64,0,1,float16,fp8,0,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,64,0,1,fp8,fp8,0,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,64,128,1,float16,float16,0,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,64,0,1,float16,float16,0,0.022495999932289124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,64,128,1,float16,fp8,0,0.024858665963013966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,64,128,1,fp8,fp8,0,0.031248000760873158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,64,0,1,float16,fp8,0,0.022650666534900665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,64,0,1,fp8,fp8,0,0.021903999149799347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,64,128,1,float16,float16,0,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,64,0,1,float16,float16,0,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,64,128,1,float16,fp8,0,0.02386666586001714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,64,128,1,fp8,fp8,0,0.03012266755104065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,64,0,1,float16,fp8,0,0.022304000953833263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,64,0,1,fp8,fp8,0,0.020618667205174763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,64,128,1,float16,float16,0,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,64,0,1,float16,float16,0,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,64,128,1,float16,fp8,0,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,64,128,1,fp8,fp8,0,0.029690665503342945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,64,0,1,float16,fp8,0,0.021701333423455555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,64,0,1,fp8,fp8,0,0.02059200033545494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,64,128,1,float16,float16,0,0.022266666094462078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,64,0,1,fp8,fp8,0,0.02029866725206375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,64,0,1,float16,float16,0,0.020714666694402695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,64,128,1,float16,fp8,0,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,64,128,1,fp8,fp8,0,0.028517333169778187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,64,128,1,fp8,fp8,0,0.02900800108909607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,64,0,1,float16,fp8,0,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,64,128,1,float16,float16,0,0.022677332162857056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,64,0,1,float16,float16,0,0.02029866725206375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,64,128,1,float16,float16,0,0.03452266752719879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,64,128,1,float16,fp8,0,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,64,0,1,float16,fp8,0,0.020666666328907013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,64,0,1,fp8,fp8,0,0.020037333170572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,64,0,1,fp8,fp8,0,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,64,0,1,float16,float16,0,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,64,128,1,float16,fp8,0,0.03450666616360346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,64,128,1,fp8,fp8,0,0.034874667723973594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,64,0,1,float16,fp8,0,0.03392533212900162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,64,128,1,float16,float16,0,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,64,0,1,float16,float16,0,0.021738665799299877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,64,128,1,float16,fp8,0,0.022240000466505688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,64,128,1,fp8,fp8,0,0.024842667082945507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,64,0,1,float16,fp8,0,0.022085333863894146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,64,0,1,fp8,fp8,0,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,64,128,1,float16,float16,0,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,64,0,1,float16,float16,0,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,64,128,1,float16,fp8,0,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,64,128,1,fp8,fp8,0,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,64,0,1,float16,fp8,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,64,0,1,fp8,fp8,0,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,64,0,1,fp8,fp8,0,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,64,128,1,float16,float16,0,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,64,0,1,float16,float16,0,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,64,128,1,float16,fp8,0,0.016058667252461117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,64,128,1,fp8,fp8,0,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,64,0,1,float16,fp8,0,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,64,128,1,float16,float16,0,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,64,0,1,float16,float16,0,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,64,128,1,float16,fp8,0,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,64,128,1,fp8,fp8,0,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,64,0,1,float16,fp8,0,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,64,0,1,fp8,fp8,0,0.017946666727463405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,64,128,1,float16,float16,0,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,64,0,1,float16,float16,0,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,64,128,1,float16,fp8,0,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,64,128,1,fp8,fp8,0,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,64,0,1,float16,fp8,0,0.014725333700577417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,64,0,1,fp8,fp8,0,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,64,128,1,float16,float16,0,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,64,0,1,float16,float16,0,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,64,128,1,float16,fp8,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,64,128,1,fp8,fp8,0,0.02075200031201045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,64,0,1,float16,fp8,0,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,64,0,1,fp8,fp8,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,64,128,1,float16,float16,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,64,0,1,float16,float16,0,0.01422400027513504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,64,128,1,float16,fp8,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,64,128,1,fp8,fp8,0,0.02070933332045873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,64,0,1,float16,fp8,0,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,64,0,1,fp8,fp8,0,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,64,128,1,float16,float16,0,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,64,0,1,float16,float16,0,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,64,128,1,float16,fp8,0,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,64,128,1,fp8,fp8,0,0.021573332448800404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,64,128,1,fp8,fp8,0,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,64,0,1,float16,fp8,0,0.019434666881958645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,64,0,1,fp8,fp8,0,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,64,128,1,float16,float16,0,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,64,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,64,128,1,float16,fp8,0,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,64,0,1,float16,fp8,0,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,64,128,1,fp8,fp8,0,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,64,0,1,fp8,fp8,0,0.019541333119074505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,64,128,1,float16,float16,0,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,64,0,1,float16,float16,0,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,64,128,1,float16,fp8,0,0.01369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,64,128,1,float16,fp8,0,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,64,0,1,float16,fp8,0,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,64,0,1,fp8,fp8,0,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,64,0,1,fp8,fp8,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,64,128,1,float16,float16,0,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,64,0,1,float16,float16,0,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,64,128,1,fp8,fp8,0,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,64,0,1,float16,fp8,0,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,64,128,1,float16,float16,0,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,64,0,1,float16,float16,0,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,64,128,1,float16,fp8,0,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,64,128,1,fp8,fp8,0,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,64,0,1,float16,fp8,0,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,64,0,1,fp8,fp8,0,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,64,128,1,float16,float16,0,0.012485332787036896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,64,0,1,float16,float16,0,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,64,128,1,float16,fp8,0,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,64,128,1,fp8,fp8,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,64,0,1,float16,fp8,0,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,64,0,1,fp8,fp8,0,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,64,128,1,float16,float16,0,0.012202666451533636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,64,0,1,float16,float16,0,0.012469333906968435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,64,128,1,float16,fp8,0,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,64,128,1,fp8,fp8,0,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,64,0,1,float16,fp8,0,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,64,0,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,64,128,1,float16,float16,0,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,64,0,1,float16,float16,0,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,64,128,1,float16,fp8,0,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,64,128,1,fp8,fp8,0,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,64,0,1,float16,fp8,0,0.01251199965675672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,64,0,1,fp8,fp8,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,64,128,1,float16,float16,0,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,64,0,1,float16,float16,0,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,64,128,1,float16,fp8,0,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,64,128,1,fp8,fp8,0,0.019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,64,0,1,float16,fp8,0,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,64,0,1,fp8,fp8,0,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,64,128,1,float16,float16,0,0.012533333152532578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,64,0,1,float16,float16,0,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,64,128,1,float16,fp8,0,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,64,128,1,fp8,fp8,0,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,64,128,1,fp8,fp8,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,64,0,1,float16,fp8,0,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,64,0,1,fp8,fp8,0,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,64,128,1,float16,float16,0,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,64,0,1,float16,float16,0,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,64,128,1,float16,fp8,0,0.012351999680201212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,64,0,1,float16,fp8,0,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,64,0,1,fp8,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,64,128,1,float16,float16,0,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,64,0,1,float16,float16,0,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,64,128,1,float16,fp8,0,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,64,128,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,64,128,1,fp8,fp8,0,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,64,0,1,float16,fp8,0,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,64,0,1,fp8,fp8,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,64,128,1,float16,float16,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,64,0,1,float16,float16,0,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,64,128,1,float16,fp8,0,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,64,0,1,float16,fp8,0,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,64,0,1,fp8,fp8,0,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,64,128,1,float16,float16,0,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,64,0,1,float16,float16,0,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,64,0,1,float16,float16,0,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,64,128,1,float16,fp8,0,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,64,128,1,fp8,fp8,0,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,64,0,1,float16,fp8,0,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,64,0,1,fp8,fp8,0,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,64,128,1,float16,float16,0,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,64,128,1,float16,fp8,0,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,64,128,1,fp8,fp8,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,64,0,1,float16,fp8,0,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,64,0,1,fp8,fp8,0,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,64,128,1,float16,float16,0,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,64,0,1,float16,float16,0,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,64,128,1,float16,fp8,0,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,64,128,1,fp8,fp8,0,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,64,0,1,float16,fp8,0,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,64,0,1,fp8,fp8,0,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,64,128,1,float16,float16,0,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,64,0,1,float16,float16,0,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,64,128,1,float16,fp8,0,0.01257066677014033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,64,128,1,fp8,fp8,0,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,64,0,1,float16,fp8,0,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,64,0,1,fp8,fp8,0,0.0185759998857975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,64,128,1,float16,float16,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,64,0,1,float16,float16,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,64,128,1,float16,fp8,0,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,64,128,1,fp8,fp8,0,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,64,0,1,float16,fp8,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,64,0,1,fp8,fp8,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,64,128,1,float16,float16,0,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,64,0,1,float16,float16,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,64,128,1,float16,fp8,0,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,64,128,1,fp8,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,64,0,1,float16,fp8,0,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,64,0,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,64,0,1,float16,fp8,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,64,128,1,float16,float16,0,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,64,0,1,float16,float16,0,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,64,128,1,float16,fp8,0,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,64,128,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,64,0,1,fp8,fp8,0,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,64,128,1,float16,float16,0,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,64,0,1,float16,float16,0,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,64,128,1,float16,fp8,0,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,64,128,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,64,0,1,float16,fp8,0,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,64,0,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,64,128,1,float16,float16,0,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,64,0,1,float16,float16,0,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,64,128,1,float16,fp8,0,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,64,128,1,fp8,fp8,0,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,64,128,1,float16,fp8,0,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,64,0,1,float16,fp8,0,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,64,0,1,fp8,fp8,0,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,64,128,1,float16,float16,0,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,64,0,1,float16,float16,0,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,64,128,1,fp8,fp8,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,64,0,1,float16,fp8,0,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,64,0,1,fp8,fp8,0,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,64,128,1,float16,float16,0,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,64,0,1,float16,float16,0,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,64,128,1,float16,fp8,0,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,64,128,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,64,0,1,float16,fp8,0,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,64,128,1,float16,fp8,0,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,64,0,1,fp8,fp8,0,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,64,128,1,float16,float16,0,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,64,0,1,float16,float16,0,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,64,128,1,fp8,fp8,0,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,64,0,1,float16,fp8,0,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,64,128,1,float16,float16,0,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,64,0,1,fp8,fp8,0,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,64,0,1,float16,float16,0,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,64,128,1,float16,fp8,0,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,64,128,1,fp8,fp8,0,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,64,0,1,float16,fp8,0,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,64,0,1,fp8,fp8,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,64,128,1,float16,float16,0,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,64,0,1,float16,float16,0,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,64,128,1,float16,float16,0,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,64,128,1,float16,fp8,0,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,64,128,1,fp8,fp8,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,64,0,1,float16,fp8,0,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,64,0,1,fp8,fp8,0,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,64,0,1,float16,float16,0,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,64,128,1,float16,fp8,0,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,64,128,1,fp8,fp8,0,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,64,128,1,fp8,fp8,0,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,64,0,1,float16,fp8,0,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,64,0,1,float16,fp8,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,64,0,1,fp8,fp8,0,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,64,128,1,float16,float16,0,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,64,0,1,float16,float16,0,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,64,128,1,float16,fp8,0,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,64,0,1,fp8,fp8,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,64,128,1,float16,float16,0,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,64,0,1,float16,float16,0,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,64,128,1,float16,fp8,0,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,64,128,1,fp8,fp8,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,64,0,1,float16,fp8,0,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,64,0,1,fp8,fp8,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,64,128,1,float16,float16,0,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,64,0,1,float16,float16,0,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,64,128,1,float16,fp8,0,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,64,128,1,fp8,fp8,0,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,64,0,1,float16,fp8,0,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,64,0,1,fp8,fp8,0,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,64,128,1,float16,float16,0,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,64,0,1,float16,float16,0,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,64,128,1,float16,fp8,0,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,64,128,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,64,0,1,float16,fp8,0,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,64,0,1,fp8,fp8,0,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,128,0,1,fp8,fp8,0,71.49880981445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,128,0,1,fp8,fp8,0,71.28865051269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,128,0,1,fp8,fp8,0,71.40462748209636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,128,0,1,fp8,fp8,0,65.48766581217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,128,0,1,fp8,fp8,0,34.63267262776693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,128,0,1,float16,fp8,0,51.08100382486979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,128,0,1,float16,float16,0,50.19896443684896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,128,0,1,float16,fp8,0,50.65904235839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,128,0,1,float16,fp8,0,50.219207763671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,128,0,1,float16,float16,0,51.07776387532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,128,0,1,float16,float16,0,50.38152567545573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,128,0,1,fp8,fp8,0,33.89604695638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,128,0,1,fp8,fp8,0,34.75269317626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,128,0,1,fp8,fp8,0,35.04502868652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,128,0,1,float16,float16,0,49.73284403483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,128,0,1,float16,float16,0,25.51897684733073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,128,0,1,float16,fp8,0,49.544352213541664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,128,0,1,float16,float16,0,49.84192911783854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,128,0,1,float16,fp8,0,50.47663370768229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,128,0,1,fp8,fp8,0,17.452767690022785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,128,0,1,float16,fp8,0,26.818682352701824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,128,0,1,fp8,fp8,0,17.538922627766926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,128,0,1,float16,float16,0,26.483067830403645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,128,0,1,float16,fp8,0,25.413108825683594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,128,0,1,fp8,fp8,0,18.526874542236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,128,0,1,float16,float16,0,25.935401916503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,128,0,1,float16,fp8,0,26.50574493408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,128,0,1,fp8,fp8,0,17.089684804280598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,128,0,1,float16,float16,0,24.949172973632812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,128,0,1,float16,fp8,0,24.98468780517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,128,0,1,float16,float16,0,13.580757141113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,128,0,1,fp8,fp8,0,17.176959991455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,128,0,1,fp8,fp8,0,10.04748789469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,128,0,1,float16,fp8,0,13.713520050048828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,128,0,1,float16,float16,0,26.73950958251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,128,0,1,float16,fp8,0,25.63330586751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,128,0,1,float16,float16,0,13.478747049967447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,128,0,1,fp8,fp8,0,8.927290598551432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,128,0,1,float16,fp8,0,13.316570281982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,128,0,1,fp8,fp8,0,8.614400227864584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,128,0,1,float16,float16,0,12.703056335449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,128,0,1,float16,fp8,0,12.70077896118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,128,0,1,float16,float16,0,13.44272486368815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,128,0,1,fp8,fp8,0,8.933637619018555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,128,0,1,float16,fp8,0,12.927018483479818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,128,0,1,fp8,fp8,0,8.603285471598307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,128,0,1,float16,fp8,0,12.778602600097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,128,0,1,float16,float16,0,13.186058044433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,128,0,1,fp8,fp8,0,39.686927795410156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,128,0,1,fp8,fp8,0,39.06387837727865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,128,0,1,float16,float16,0,59.1903076171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,128,0,1,float16,fp8,0,59.1585693359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,128,0,1,float16,float16,0,58.19965108235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,128,0,1,float16,fp8,0,60.78961181640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,128,0,1,float16,float16,0,59.88211568196615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,128,0,1,fp8,fp8,0,39.606432596842446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,128,0,1,float16,float16,0,31.720352172851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,128,0,1,float16,fp8,0,58.86582946777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,128,0,1,float16,fp8,0,30.731524149576824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,128,0,1,fp8,fp8,0,40.22476704915365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,128,0,1,fp8,fp8,0,20.776971181233723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,128,0,1,float16,float16,0,58.294148763020836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,128,0,1,float16,fp8,0,59.65821329752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,128,0,1,fp8,fp8,0,20.439652760823567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,128,0,1,float16,float16,0,29.51903025309245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,128,0,1,float16,fp8,0,29.213760375976562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,128,0,1,float16,float16,0,29.48138173421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,128,0,1,fp8,fp8,0,20.388421376546223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,128,0,1,float16,fp8,0,29.95068359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,128,0,1,fp8,fp8,0,19.951274871826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,128,0,1,float16,float16,0,29.80890146891276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,128,0,1,float16,fp8,0,29.834917704264324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,128,0,1,float16,float16,0,15.30886967976888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,128,0,1,fp8,fp8,0,20.176005045572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,128,0,1,float16,float16,0,29.06072998046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,128,0,1,fp8,fp8,0,10.553573608398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,128,0,1,float16,fp8,0,29.061065673828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,128,0,1,float16,fp8,0,15.66876220703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,128,0,1,float16,float16,0,15.026554107666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,128,0,1,fp8,fp8,0,10.100725173950195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,128,0,1,fp8,fp8,0,10.259669621785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,128,0,1,float16,fp8,0,15.136895497639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,128,0,1,float16,float16,0,14.977391560872396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,128,0,1,float16,fp8,0,15.007792154947916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,128,0,1,float16,float16,0,14.862709045410156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,128,0,1,fp8,fp8,0,10.179285049438477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,128,0,1,float16,fp8,0,15.174639383951822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,128,0,1,float16,float16,0,8.087013244628906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,128,0,1,fp8,fp8,0,10.10580825805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,128,0,1,fp8,fp8,0,5.562447865804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,128,0,1,float16,float16,0,14.707195281982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,128,0,1,float16,fp8,0,14.841578165690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,128,0,1,float16,fp8,0,7.7576853434244795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,128,0,1,float16,float16,0,7.862000147501628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,128,0,1,fp8,fp8,0,5.17794672648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,128,0,1,float16,fp8,0,7.719013214111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,128,0,1,fp8,fp8,0,5.265647888183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,128,0,1,float16,float16,0,7.3869279225667315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,128,0,1,float16,fp8,0,7.644704182942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,128,0,1,fp8,fp8,0,5.1844587326049805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,128,0,1,float16,float16,0,7.608554840087891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,128,0,1,float16,fp8,0,7.835856119791667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,128,0,1,float16,float16,0,7.802725474039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,128,0,1,fp8,fp8,0,5.370496114095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,128,0,1,float16,fp8,0,7.757717132568359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,128,0,1,fp8,fp8,0,28.26246388753255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,128,0,1,fp8,fp8,0,28.334505716959637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,128,0,1,float16,float16,0,41.37896474202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,128,0,1,float16,fp8,0,41.84223429361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,128,0,1,float16,float16,0,42.20441691080729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,128,0,1,float16,fp8,0,41.73406473795573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,128,0,1,float16,fp8,0,46.626688639322914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,128,0,1,float16,float16,0,46.15375264485677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,128,0,1,fp8,fp8,0,28.99609629313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,128,0,1,float16,float16,0,22.067476908365887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,128,0,1,float16,fp8,0,22.085001627604168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,128,0,1,fp8,fp8,0,28.547444661458332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,128,0,1,fp8,fp8,0,14.553642272949219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,128,0,1,float16,float16,0,42.04796854654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,128,0,1,float16,float16,0,20.012645721435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,128,0,1,float16,fp8,0,20.660906473795574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,128,0,1,float16,fp8,0,41.75178782145182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,128,0,1,fp8,fp8,0,14.51864496866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,128,0,1,fp8,fp8,0,14.827279408772787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,128,0,1,float16,float16,0,20.93996810913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,128,0,1,float16,fp8,0,20.300538380940754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,128,0,1,fp8,fp8,0,14.75927480061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,128,0,1,float16,float16,0,21.113680521647137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,128,0,1,float16,float16,0,11.10251235961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,128,0,1,float16,fp8,0,20.420608520507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,128,0,1,fp8,fp8,0,7.561567942301433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,128,0,1,float16,float16,0,21.097979227701824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,128,0,1,fp8,fp8,0,14.074479420979818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,128,0,1,float16,fp8,0,21.74120585123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,128,0,1,float16,fp8,0,10.839711507161459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,128,0,1,float16,float16,0,10.688971201578775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,128,0,1,fp8,fp8,0,6.9436798095703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,128,0,1,fp8,fp8,0,7.091759999593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,128,0,1,float16,fp8,0,10.603776295979818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,128,0,1,float16,float16,0,10.690149943033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,128,0,1,float16,fp8,0,10.471018473307291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,128,0,1,float16,float16,0,10.944843292236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,128,0,1,float16,fp8,0,10.709279378255209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,128,0,1,fp8,fp8,0,7.28495979309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,128,0,1,float16,float16,0,5.692261377970378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,128,0,1,fp8,fp8,0,4.029066721598308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,128,0,1,float16,fp8,0,5.609381357828776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,128,0,1,fp8,fp8,0,7.221498489379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,128,0,1,float16,float16,0,10.526992162068685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,128,0,1,float16,float16,0,5.350170771280925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,128,0,1,float16,fp8,0,10.566645304361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,128,0,1,float16,fp8,0,5.476570765177409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,128,0,1,fp8,fp8,0,3.676959991455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,128,0,1,fp8,fp8,0,3.770517349243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,128,0,1,float16,float16,0,5.521674474080403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,128,0,1,float16,fp8,0,5.519594828287761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,128,0,1,fp8,fp8,0,3.9146931966145835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,128,0,1,float16,float16,0,5.386533101399739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,128,0,1,float16,fp8,0,5.602415720621745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,128,0,1,fp8,fp8,0,3.722522735595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,128,0,1,float16,float16,0,5.521978378295898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,128,0,1,float16,fp8,0,6.169141133626302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,128,0,1,fp8,fp8,0,36.28291320800781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,128,0,1,fp8,fp8,0,36.38039906819662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,128,0,1,fp8,fp8,0,36.7958984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,128,0,1,fp8,fp8,0,40.93389892578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,128,0,1,float16,float16,0,27.437728881835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,128,0,1,float16,fp8,0,27.36762237548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,128,0,1,fp8,fp8,0,18.266053517659504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,128,0,1,fp8,fp8,0,18.257476806640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,128,0,1,float16,fp8,0,26.205434163411457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,128,0,1,float16,float16,0,26.200164794921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,128,0,1,float16,float16,0,26.724688212076824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,128,0,1,float16,fp8,0,26.602457682291668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,128,0,1,fp8,fp8,0,20.796778361002605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,128,0,1,float16,float16,0,26.836219787597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,128,0,1,float16,float16,0,14.057989756266275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,128,0,1,float16,fp8,0,27.947962443033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,128,0,1,fp8,fp8,0,18.399285634358723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,128,0,1,fp8,fp8,0,9.904607772827148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,128,0,1,float16,fp8,0,14.139060974121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,128,0,1,float16,fp8,0,26.464820861816406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,128,0,1,float16,float16,0,26.548052469889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,128,0,1,fp8,fp8,0,9.16540273030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,128,0,1,float16,float16,0,13.363381703694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,128,0,1,float16,fp8,0,13.439797719319662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,128,0,1,fp8,fp8,0,10.051984151204428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,128,0,1,float16,float16,0,13.557034810384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,128,0,1,float16,fp8,0,13.331546783447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,128,0,1,float16,float16,0,14.028772989908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,128,0,1,fp8,fp8,0,10.236661275227865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,128,0,1,float16,fp8,0,14.407301584879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,128,0,1,fp8,fp8,0,5.226037343343099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,128,0,1,fp8,fp8,0,9.335823694864908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,128,0,1,float16,float16,0,13.688218434651693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,128,0,1,float16,fp8,0,7.287290573120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,128,0,1,float16,float16,0,7.387184143066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,128,0,1,float16,fp8,0,13.710890452067057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,128,0,1,float16,float16,0,7.048816045125325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,128,0,1,fp8,fp8,0,4.762997309366862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,128,0,1,float16,fp8,0,6.986586888631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,128,0,1,fp8,fp8,0,4.775386810302734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,128,0,1,float16,float16,0,6.686469395955403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,128,0,1,float16,fp8,0,6.8188050587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,128,0,1,float16,float16,0,6.883786519368489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,128,0,1,fp8,fp8,0,4.652154604593913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,128,0,1,float16,fp8,0,7.147082646687825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,128,0,1,fp8,fp8,0,4.644469261169434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,128,0,1,fp8,fp8,0,2.6076906522115073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,128,0,1,float16,float16,0,6.646575927734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,128,0,1,float16,fp8,0,7.09108289082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,128,0,1,float16,fp8,0,3.7613118489583335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,128,0,1,float16,float16,0,3.598442713419596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,128,0,1,float16,float16,0,3.709679921468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,128,0,1,fp8,fp8,0,2.46016534169515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,128,0,1,float16,fp8,0,3.4656852086385093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,128,0,1,fp8,fp8,0,2.4865546226501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,128,0,1,float16,float16,0,3.4767786661783853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,128,0,1,float16,fp8,0,3.432842572530111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,128,0,1,fp8,fp8,0,2.4314986864725747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,128,0,1,float16,float16,0,3.4753281275431314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,128,0,1,float16,fp8,0,3.6179787317911782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,128,0,1,float16,float16,0,3.5344107945760093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,128,0,1,fp8,fp8,0,2.556309382120768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,128,0,1,float16,fp8,0,3.7394933700561523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,128,0,1,fp8,fp8,0,21.82202657063802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,128,0,1,fp8,fp8,0,21.442100524902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,128,0,1,fp8,fp8,0,23.031023661295574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,128,0,1,fp8,fp8,0,21.533348083496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,128,0,1,float16,float16,0,16.335520426432293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,128,0,1,float16,fp8,0,17.23576482137044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,128,0,1,fp8,fp8,0,10.724533081054688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,128,0,1,fp8,fp8,0,11.701034545898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,128,0,1,float16,float16,0,15.410650889078775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,128,0,1,float16,float16,0,15.535877227783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,128,0,1,float16,fp8,0,15.70236841837565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,128,0,1,float16,fp8,0,15.652671813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,128,0,1,fp8,fp8,0,11.069578806559244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,128,0,1,fp8,fp8,0,10.9650026957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,128,0,1,float16,fp8,0,15.529195149739584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,128,0,1,fp8,fp8,0,10.7281862894694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,128,0,1,float16,float16,0,15.505023956298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,128,0,1,float16,float16,0,16.82306162516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,128,0,1,float16,fp8,0,16.71564229329427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,128,0,1,float16,float16,0,8.220762888590494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,128,0,1,fp8,fp8,0,7.2198028564453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,128,0,1,float16,float16,0,9.427973429361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,128,0,1,float16,fp8,0,9.505872090657553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,128,0,1,float16,fp8,0,8.393722534179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,128,0,1,fp8,fp8,0,5.78443717956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,128,0,1,float16,fp8,0,7.876842498779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,128,0,1,float16,float16,0,7.5774186452229815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,128,0,1,fp8,fp8,0,5.643141428629558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,128,0,1,fp8,fp8,0,5.502389272054036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,128,0,1,float16,float16,0,7.8001708984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,128,0,1,float16,fp8,0,7.953226725260417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,128,0,1,float16,float16,0,7.877072016398112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,128,0,1,fp8,fp8,0,5.671834945678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,128,0,1,float16,float16,0,4.164714813232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,128,0,1,float16,fp8,0,8.225151697794596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,128,0,1,fp8,fp8,0,3.0170933405558267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,128,0,1,float16,fp8,0,4.613242785135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,128,0,1,fp8,fp8,0,2.7293386459350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,128,0,1,float16,fp8,0,4.168426513671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,128,0,1,float16,float16,0,3.998506546020508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,128,0,1,float16,float16,0,4.046522776285808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,128,0,1,fp8,fp8,0,2.819093386332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,128,0,1,float16,fp8,0,4.134085337320964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,128,0,1,float16,float16,0,4.083834648132324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,128,0,1,fp8,fp8,0,2.814085324605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,128,0,1,float16,fp8,0,4.152138710021973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,128,0,1,float16,float16,0,3.9681545893351235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,128,0,1,fp8,fp8,0,2.809328079223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,128,0,1,float16,float16,0,2.2294346491495767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,128,0,1,float16,fp8,0,2.1538772583007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,128,0,1,float16,fp8,0,4.183722813924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,128,0,1,float16,float16,0,2.012335936228434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,128,0,1,fp8,fp8,0,1.5611146291097004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,128,0,1,fp8,fp8,0,1.4746665954589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,128,0,1,float16,fp8,0,2.0061119397481284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,128,0,1,fp8,fp8,0,1.4856692949930828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,128,0,1,float16,fp8,0,2.0472853978474936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,128,0,1,float16,float16,0,2.060688018798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,128,0,1,float16,float16,0,2.015530745188395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,128,0,1,fp8,fp8,0,1.4772586822509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,128,0,1,float16,fp8,0,2.0746560096740723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,128,0,1,float16,float16,0,2.0969866116841636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,128,0,1,fp8,fp8,0,1.482693354288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,128,0,1,float16,fp8,0,2.1942826906840005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,128,0,1,fp8,fp8,0,21.45880126953125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,128,0,1,fp8,fp8,0,21.456436157226562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,128,0,1,fp8,fp8,0,21.574310302734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,128,0,1,fp8,fp8,0,21.55847422281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,128,0,1,float16,float16,0,16.287370045979817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,128,0,1,float16,fp8,0,16.922640482584637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,128,0,1,float16,float16,0,14.954442342122396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,128,0,1,float16,fp8,0,14.96896489461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,128,0,1,fp8,fp8,0,11.103711446126303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,128,0,1,float16,float16,0,14.97979736328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,128,0,1,float16,fp8,0,15.009877522786459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,128,0,1,fp8,fp8,0,10.716047922770182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,128,0,1,float16,float16,0,14.99884287516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,128,0,1,float16,fp8,0,15.321669260660807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,128,0,1,float16,fp8,0,8.128655751546225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,128,0,1,float16,float16,0,8.078549067179361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,128,0,1,fp8,fp8,0,10.65728505452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,128,0,1,fp8,fp8,0,10.68939208984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,128,0,1,float16,float16,0,7.342842737833659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,128,0,1,float16,fp8,0,7.555440266927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,128,0,1,float16,float16,0,15.087610880533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,128,0,1,float16,fp8,0,15.113722483317057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,128,0,1,fp8,fp8,0,5.319525400797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,128,0,1,fp8,fp8,0,5.836661020914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,128,0,1,fp8,fp8,0,6.037839889526367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,128,0,1,float16,float16,0,7.629674911499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,128,0,1,fp8,fp8,0,5.4087785085042315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,128,0,1,float16,float16,0,7.512175877888997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,128,0,1,float16,fp8,0,7.497653325398763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,128,0,1,float16,fp8,0,9.259520212809244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,128,0,1,float16,float16,0,7.427077611287435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,128,0,1,fp8,fp8,0,5.354031880696614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,128,0,1,fp8,fp8,0,2.888554573059082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,128,0,1,float16,float16,0,3.7889973322550454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,128,0,1,float16,fp8,0,7.684458414713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,128,0,1,float16,float16,0,4.129839897155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,128,0,1,fp8,fp8,0,2.6881545384724936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,128,0,1,float16,fp8,0,4.173845291137695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,128,0,1,float16,fp8,0,3.9097973505655923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,128,0,1,float16,fp8,0,3.932352066040039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,128,0,1,fp8,fp8,0,2.7790985107421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,128,0,1,float16,float16,0,3.5979092915852866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,128,0,1,fp8,fp8,0,2.9805332819620767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,128,0,1,float16,float16,0,3.814490636189779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,128,0,1,float16,fp8,0,4.7873334884643555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,128,0,1,float16,float16,0,3.7032159169514975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,128,0,1,fp8,fp8,0,2.8693758646647134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,128,0,1,float16,float16,0,2.130842685699463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,128,0,1,fp8,fp8,0,1.6500800450642903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,128,0,1,float16,fp8,0,2.1731413205464682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,128,0,1,float16,fp8,0,3.9467360178629556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,128,0,1,float16,float16,0,2.149498621622721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,128,0,1,float16,fp8,0,1.8684266408284504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,128,0,1,fp8,fp8,0,1.4297812779744465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,128,0,1,fp8,fp8,0,1.4229332605997722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,128,0,1,float16,float16,0,2.12225071589152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,128,0,1,float16,fp8,0,1.9806186358133953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,128,0,1,float16,float16,0,1.8689173062642415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,128,0,1,fp8,fp8,0,1.4061439832051594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,128,0,1,float16,fp8,0,1.896399974822998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,128,0,1,float16,float16,0,1.8518560727437336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,128,0,1,fp8,fp8,0,1.4185706774393718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,128,0,1,float16,float16,0,1.0648319721221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,128,0,1,float16,fp8,0,1.8810720443725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,128,0,1,float16,fp8,0,1.0719359715779622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,128,0,1,float16,float16,0,1.0246506532033284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,128,0,1,float16,fp8,0,1.0280106862386067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,128,0,1,float16,float16,0,1.0313706398010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,128,0,1,fp8,fp8,0,0.7872320016225179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,128,0,1,fp8,fp8,0,0.832810640335083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,128,0,1,float16,fp8,0,1.0455466906229656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,128,0,1,fp8,fp8,0,0.7812053362528483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,128,0,1,fp8,fp8,0,0.7929920355478922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,128,0,1,float16,float16,0,1.0293227036794026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,128,0,1,float16,float16,0,1.0154826641082764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,128,0,1,float16,fp8,0,1.0334933598836262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,128,0,1,fp8,fp8,0,0.8065813382466634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,128,0,1,float16,fp8,0,1.0344959894816081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,128,0,1,fp8,fp8,0,13.255471547444662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,128,0,1,fp8,fp8,0,13.475589752197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,128,0,1,float16,fp8,0,18.721829732259113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,128,0,1,fp8,fp8,0,13.471343994140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,128,0,1,fp8,fp8,0,13.477444966634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,128,0,1,float16,float16,0,9.90998395284017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,128,0,1,float16,fp8,0,10.178288141886393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,128,0,1,fp8,fp8,0,7.443215688069661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,128,0,1,float16,float16,0,9.50981330871582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,128,0,1,float16,fp8,0,9.16158930460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,128,0,1,fp8,fp8,0,6.887200037638347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,128,0,1,float16,float16,0,9.197216033935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,128,0,1,float16,fp8,0,9.353167851765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,128,0,1,fp8,fp8,0,6.730661392211914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,128,0,1,float16,float16,0,9.199717203776041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,128,0,1,float16,fp8,0,9.422362645467123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,128,0,1,fp8,fp8,0,7.061269124348958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,128,0,1,float16,float16,0,9.384501139322916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,128,0,1,float16,fp8,0,9.31936009724935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,128,0,1,fp8,fp8,0,6.82749875386556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,128,0,1,float16,float16,0,5.08021863301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,128,0,1,float16,fp8,0,4.987301190694173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,128,0,1,fp8,fp8,0,3.706965446472168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,128,0,1,float16,float16,0,4.468026796976726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,128,0,1,float16,fp8,0,4.544015884399414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,128,0,1,fp8,fp8,0,3.3524961471557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,128,0,1,float16,float16,0,4.494304021199544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,128,0,1,float16,fp8,0,4.420421282450358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,128,0,1,fp8,fp8,0,3.2191572189331055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,128,0,1,float16,float16,0,4.6142934163411455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,128,0,1,float16,fp8,0,4.619957288106282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,128,0,1,fp8,fp8,0,3.205098787943522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,128,0,1,float16,fp8,0,4.783002535502116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,128,0,1,float16,float16,0,4.552778561909993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,128,0,1,fp8,fp8,0,3.2476374308268228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,128,0,1,float16,float16,0,2.420815944671631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,128,0,1,float16,fp8,0,2.4565439224243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,128,0,1,fp8,fp8,0,1.8953065872192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,128,0,1,float16,float16,0,2.1842880249023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,128,0,1,float16,fp8,0,2.1974453926086426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,128,0,1,float16,fp8,0,2.290309270222982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,128,0,1,fp8,fp8,0,1.6970613797505696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,128,0,1,float16,float16,0,2.261829376220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,128,0,1,fp8,fp8,0,1.6673653920491536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,128,0,1,float16,float16,0,2.1707839965820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,128,0,1,float16,fp8,0,2.1920906702677407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,128,0,1,fp8,fp8,0,1.672346591949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,128,0,1,float16,fp8,0,2.2013813654581704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,128,0,1,float16,float16,0,1.2610666751861572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,128,0,1,float16,float16,0,2.17958927154541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,128,0,1,fp8,fp8,0,1.693552017211914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,128,0,1,float16,fp8,0,1.2531840006510417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,128,0,1,fp8,fp8,0,1.0035466353098552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,128,0,1,float16,float16,0,1.1769386927286785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,128,0,1,float16,fp8,0,1.1546400388081868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,128,0,1,fp8,fp8,0,0.9122453530629476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,128,0,1,float16,float16,0,1.163802703221639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,128,0,1,fp8,fp8,0,0.913914680480957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,128,0,1,float16,fp8,0,1.1509973208109539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,128,0,1,float16,float16,0,1.1655413309733074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,128,0,1,fp8,fp8,0,0.902890682220459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,128,0,1,float16,fp8,0,1.1669387022654216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,128,0,1,float16,float16,0,1.1666453679402669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,128,0,1,float16,fp8,0,1.1699893474578857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,128,0,1,fp8,fp8,0,0.9019253253936768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,128,0,1,float16,float16,0,0.6797013282775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,128,0,1,fp8,fp8,0,0.5577013492584229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,128,0,1,float16,float16,0,0.6438613335291544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,128,0,1,float16,fp8,0,0.6845973332722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,128,0,1,fp8,fp8,0,0.5192213455835978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,128,0,1,float16,float16,0,0.6538240114847819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,128,0,1,float16,fp8,0,0.6482613484064738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,128,0,1,float16,fp8,0,0.6491413513819376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,128,0,1,fp8,fp8,0,0.524837334950765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,128,0,1,float16,float16,0,0.6527040004730225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,128,0,1,float16,fp8,0,0.6521120071411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,128,0,1,fp8,fp8,0,0.526095986366272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,128,0,1,float16,float16,0,0.6482719977696737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,128,0,1,float16,fp8,0,0.657477339108785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,128,0,1,fp8,fp8,0,0.5277440150578817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,128,0,1,fp8,fp8,0,14.123402913411459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,128,0,1,fp8,fp8,0,13.989402770996094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,128,0,1,fp8,fp8,0,14.162373860677084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,128,0,1,fp8,fp8,0,14.30297597249349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,128,0,1,float16,float16,0,10.738180796305338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,128,0,1,float16,float16,0,9.078261057535807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,128,0,1,float16,fp8,0,10.300005594889322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,128,0,1,fp8,fp8,0,7.03218142191569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,128,0,1,float16,fp8,0,9.369845072428385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,128,0,1,float16,float16,0,9.263504028320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,128,0,1,float16,fp8,0,9.528565088907877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,128,0,1,fp8,fp8,0,7.065781275431315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,128,0,1,float16,float16,0,9.41865094502767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,128,0,1,fp8,fp8,0,7.110495885213216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,128,0,1,float16,float16,0,9.476687749226889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,128,0,1,fp8,fp8,0,7.240933100382487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,128,0,1,float16,fp8,0,9.370138804117838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,128,0,1,fp8,fp8,0,4.0182294845581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,128,0,1,float16,float16,0,4.535077412923177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,128,0,1,float16,fp8,0,9.310272216796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,128,0,1,float16,float16,0,5.310208002726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,128,0,1,float16,fp8,0,5.234629313151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,128,0,1,float16,fp8,0,4.564976056416829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,128,0,1,fp8,fp8,0,3.4032106399536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,128,0,1,fp8,fp8,0,3.4385226567586265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,128,0,1,float16,fp8,0,4.448378562927246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,128,0,1,float16,float16,0,4.70194149017334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,128,0,1,float16,float16,0,4.583109219868978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,128,0,1,fp8,fp8,0,3.4794772466023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,128,0,1,float16,fp8,0,4.487589200337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,128,0,1,float16,float16,0,4.612304051717122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,128,0,1,float16,float16,0,2.587578614552816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,128,0,1,float16,fp8,0,2.5387306213378906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,128,0,1,float16,fp8,0,4.542096138000488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,128,0,1,fp8,fp8,0,2.0373546282450357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,128,0,1,fp8,fp8,0,3.4635680516560874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,128,0,1,float16,fp8,0,2.177776018778483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,128,0,1,float16,float16,0,2.2116106351216636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,128,0,1,fp8,fp8,0,1.7793067296346028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,128,0,1,float16,float16,0,2.3015732765197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,128,0,1,float16,float16,0,2.171562671661377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,128,0,1,float16,fp8,0,2.2687253952026367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,128,0,1,float16,fp8,0,2.2639733950297036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,128,0,1,fp8,fp8,0,1.7964159647623699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,128,0,1,fp8,fp8,0,1.7829705874125164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,128,0,1,float16,fp8,0,1.3034239610036213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,128,0,1,fp8,fp8,0,1.7696372667948406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,128,0,1,float16,float16,0,2.3309226036071777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,128,0,1,float16,fp8,0,2.287322680155436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,128,0,1,fp8,fp8,0,1.072266658147176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,128,0,1,float16,float16,0,1.1494399706522624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,128,0,1,float16,fp8,0,1.1411893367767334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,128,0,1,fp8,fp8,0,0.9407839775085449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,128,0,1,float16,fp8,0,1.1284053325653076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,128,0,1,fp8,fp8,0,0.9327840010325114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,128,0,1,float16,float16,0,1.1506346861521404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,128,0,1,float16,float16,0,1.3043999671936035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,128,0,1,float16,float16,0,1.1502666473388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,128,0,1,float16,fp8,0,1.1460479895273845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,128,0,1,fp8,fp8,0,0.945146640141805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,128,0,1,float16,float16,0,1.1301813125610352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,128,0,1,fp8,fp8,0,0.9392266273498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,128,0,1,float16,fp8,0,1.1508639653523762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,128,0,1,fp8,fp8,0,0.5731093486150106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,128,0,1,float16,float16,0,0.6826559702555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,128,0,1,float16,fp8,0,0.6872266928354899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,128,0,1,float16,float16,0,0.6178933382034302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,128,0,1,float16,fp8,0,0.6165279944737753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,128,0,1,fp8,fp8,0,0.5116000175476074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,128,0,1,float16,float16,0,0.6269226471583048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,128,0,1,fp8,fp8,0,0.5172693332036337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,128,0,1,float16,fp8,0,0.6238613526026408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,128,0,1,float16,float16,0,0.6293493509292603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,128,0,1,float16,fp8,0,0.6237279971440634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,128,0,1,fp8,fp8,0,0.5113226572672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,128,0,1,float16,float16,0,0.6207679907480875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,128,0,1,float16,float16,0,0.379258672396342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,128,0,1,fp8,fp8,0,0.5197759866714478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,128,0,1,float16,fp8,0,0.6339093446731567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,128,0,1,fp8,fp8,0,0.3335733413696289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,128,0,1,float16,fp8,0,0.389792005221049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,128,0,1,float16,fp8,0,0.3649333318074544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,128,0,1,float16,float16,0,0.35308265686035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,128,0,1,float16,fp8,0,0.36340800921122235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,128,0,1,fp8,fp8,0,0.2923840085665385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,128,0,1,float16,float16,0,0.36614398161570233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,128,0,1,fp8,fp8,0,0.29287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,128,0,1,float16,float16,0,0.36612268288930255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,128,0,1,float16,fp8,0,0.364303986231486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,128,0,1,fp8,fp8,0,0.2870933413505554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,128,0,1,fp8,fp8,0,0.29603199164072674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,128,0,1,float16,float16,0,0.3681120077768962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,128,0,1,float16,fp8,0,0.3669919967651367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,128,0,1,fp8,fp8,0,9.289007822672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,128,0,1,fp8,fp8,0,9.284624099731445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,128,0,1,float16,fp8,0,11.731775919596354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,128,0,1,fp8,fp8,0,9.361717224121094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,128,0,1,fp8,fp8,0,9.52461306254069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,128,0,1,float16,float16,0,6.852783838907878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,128,0,1,float16,fp8,0,6.7142079671223955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,128,0,1,fp8,fp8,0,5.468992233276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,128,0,1,float16,float16,0,5.906229019165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,128,0,1,float16,fp8,0,6.911077499389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,128,0,1,fp8,fp8,0,4.695450782775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,128,0,1,float16,float16,0,5.998250961303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,128,0,1,float16,fp8,0,5.861194610595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,128,0,1,fp8,fp8,0,4.63263479868571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,128,0,1,float16,float16,0,6.063760121663411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,128,0,1,float16,fp8,0,5.833690643310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,128,0,1,fp8,fp8,0,4.63755734761556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,128,0,1,float16,float16,0,6.043909072875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,128,0,1,float16,fp8,0,5.830399831136067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,128,0,1,fp8,fp8,0,4.648576100667317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,128,0,1,float16,float16,0,3.4104798634847007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,128,0,1,float16,fp8,0,3.5411148071289062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,128,0,1,fp8,fp8,0,2.7999146779378257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,128,0,1,float16,float16,0,2.7827253341674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,128,0,1,float16,fp8,0,2.8427305221557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,128,0,1,fp8,fp8,0,2.279973347981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,128,0,1,float16,float16,0,2.8545173009236655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,128,0,1,float16,fp8,0,2.7979040145874023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,128,0,1,fp8,fp8,0,2.292853355407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,128,0,1,float16,float16,0,2.810890515645345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,128,0,1,float16,fp8,0,2.8345600763956704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,128,0,1,fp8,fp8,0,2.2942506472269693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,128,0,1,float16,float16,0,2.9336268107096353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,128,0,1,float16,fp8,0,2.753562609354655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,128,0,1,fp8,fp8,0,2.3548107147216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,128,0,1,float16,float16,0,1.6712640126546223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,128,0,1,float16,fp8,0,1.6663306554158528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,128,0,1,fp8,fp8,0,1.4176799456278484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,128,0,1,float16,float16,0,1.3833173116048176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,128,0,1,float16,fp8,0,1.3956693013509114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,128,0,1,fp8,fp8,0,1.1933066844940186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,128,0,1,float16,float16,0,1.387056032816569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,128,0,1,float16,fp8,0,1.423093318939209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,128,0,1,fp8,fp8,0,1.207103967666626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,128,0,1,float16,fp8,0,1.4245440165201824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,128,0,1,fp8,fp8,0,1.2165813446044922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,128,0,1,float16,float16,0,1.4207146962483723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,128,0,1,float16,float16,0,1.4029760360717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,128,0,1,float16,fp8,0,1.4325226147969563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,128,0,1,float16,float16,0,0.8661386966705322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,128,0,1,fp8,fp8,0,1.1957813103993733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,128,0,1,float16,fp8,0,0.8585279782613119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,128,0,1,float16,float16,0,0.7418933709462484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,128,0,1,fp8,fp8,0,0.727242628733317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,128,0,1,float16,fp8,0,0.7437919775644938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,128,0,1,fp8,fp8,0,0.6291893323262533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,128,0,1,float16,float16,0,0.7440853118896484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,128,0,1,float16,fp8,0,0.7459039688110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,128,0,1,fp8,fp8,0,0.6406026681264242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,128,0,1,float16,float16,0,0.7440693378448486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,128,0,1,float16,fp8,0,0.7519946893056234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,128,0,1,fp8,fp8,0,0.6382346550623575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,128,0,1,float16,float16,0,0.7468852996826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,128,0,1,float16,fp8,0,0.7539626757303873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,128,0,1,fp8,fp8,0,0.6454399824142456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,128,0,1,float16,float16,0,0.45532798767089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,128,0,1,float16,fp8,0,0.45580800374348956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,128,0,1,fp8,fp8,0,0.4011146624883016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,128,0,1,float16,float16,0,0.40294933319091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,128,0,1,float16,fp8,0,0.4118933280309041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,128,0,1,fp8,fp8,0,0.3532960017522176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,128,0,1,float16,float16,0,0.4150613149007161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,128,0,1,float16,fp8,0,0.41168534755706787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,128,0,1,fp8,fp8,0,0.3560853401819865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,128,0,1,float16,float16,0,0.4091200033823649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,128,0,1,float16,fp8,0,0.4148746728897095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,128,0,1,fp8,fp8,0,0.35438934961954754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,128,0,1,float16,float16,0,0.4128640095392863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,128,0,1,float16,fp8,0,0.4133439858754476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,128,0,1,fp8,fp8,0,0.3585280179977417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,128,0,1,float16,float16,0,0.25862934192021686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,128,0,1,float16,fp8,0,0.26422399282455444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,128,0,1,fp8,fp8,0,0.2304426630338033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,128,0,1,float16,float16,0,0.24612265825271606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,128,0,1,float16,fp8,0,0.24623999993006387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,128,0,1,fp8,fp8,0,0.20349866151809692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,128,0,1,float16,float16,0,0.24505066871643066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,128,0,1,float16,fp8,0,0.24740266799926758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,128,0,1,fp8,fp8,0,0.20278932650883993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,128,0,1,float16,float16,0,0.24762133757273355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,128,0,1,float16,fp8,0,0.24798399209976196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,128,0,1,fp8,fp8,0,0.20450133085250854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,128,0,1,float16,float16,0,0.24706133206685385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,128,0,1,float16,fp8,0,0.2498133381207784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,128,0,1,fp8,fp8,0,0.20443199078241983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,128,0,1,fp8,fp8,0,10.064591725667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,128,0,1,fp8,fp8,0,10.108378728230795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,128,0,1,fp8,fp8,0,10.24027188618978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,128,0,1,float16,float16,0,7.645578384399414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,128,0,1,fp8,fp8,0,10.542165120442709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,128,0,1,float16,fp8,0,7.521370569864909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,128,0,1,float16,float16,0,5.950549443562825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,128,0,1,float16,fp8,0,5.792874654134114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,128,0,1,fp8,fp8,0,5.04912535349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,128,0,1,float16,float16,0,5.944234848022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,128,0,1,float16,fp8,0,6.027882893880208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,128,0,1,fp8,fp8,0,5.090688069661458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,128,0,1,float16,float16,0,5.98579216003418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,128,0,1,float16,fp8,0,5.958746592203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,128,0,1,fp8,fp8,0,5.068165461222331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,128,0,1,float16,float16,0,6.051557540893555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,128,0,1,float16,fp8,0,6.070538838704427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,128,0,1,float16,float16,0,3.8459466298421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,128,0,1,fp8,fp8,0,5.2964426676432295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,128,0,1,float16,fp8,0,3.7184534072875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,128,0,1,fp8,fp8,0,3.179391860961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,128,0,1,float16,float16,0,2.914378801981608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,128,0,1,fp8,fp8,0,2.522517363230387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,128,0,1,float16,float16,0,2.880314509073893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,128,0,1,float16,fp8,0,2.9276854197184243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,128,0,1,fp8,fp8,0,2.5773866971333823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,128,0,1,float16,float16,0,2.8920373916625977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,128,0,1,float16,fp8,0,2.871482531229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,128,0,1,float16,fp8,0,2.8786398569742837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,128,0,1,fp8,fp8,0,2.583029270172119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,128,0,1,float16,float16,0,2.958400090535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,128,0,1,fp8,fp8,0,2.667872111002604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,128,0,1,float16,fp8,0,2.9447145462036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,128,0,1,float16,float16,0,1.9289226531982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,128,0,1,fp8,fp8,0,1.5905493100484211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,128,0,1,float16,fp8,0,1.8399573961893718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,128,0,1,float16,float16,0,1.4537332852681477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,128,0,1,float16,fp8,0,1.4478185971577961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,128,0,1,fp8,fp8,0,1.2666773001352947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,128,0,1,float16,float16,0,1.4607092539469402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,128,0,1,float16,fp8,0,1.4694026311238606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,128,0,1,fp8,fp8,0,1.2724053064982097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,128,0,1,float16,float16,0,1.4616586367289226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,128,0,1,fp8,fp8,0,1.2861333688100178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,128,0,1,float16,fp8,0,1.4697759946187336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,128,0,1,float16,float16,0,0.9704053401947021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,128,0,1,fp8,fp8,0,0.8067946434020996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,128,0,1,float16,float16,0,0.7462773323059082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,128,0,1,float16,fp8,0,0.7454186280568441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,128,0,1,fp8,fp8,0,0.6408480008443197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,128,0,1,float16,float16,0,0.7473226388295492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,128,0,1,float16,fp8,0,0.747429370880127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,128,0,1,fp8,fp8,0,0.6478720108668009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,128,0,1,float16,float16,0,0.7475787003835043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,128,0,1,float16,fp8,0,0.7486506303151449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,128,0,1,fp8,fp8,0,0.6529759963353475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,128,0,1,float16,float16,0,1.4741919835408528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,128,0,1,fp8,fp8,0,1.357242743174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,128,0,1,float16,float16,0,0.7525653044382731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,128,0,1,float16,fp8,0,0.7534399827321371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,128,0,1,float16,fp8,0,1.4672800699869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,128,0,1,float16,fp8,0,0.9280426502227783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,128,0,1,fp8,fp8,0,0.6899413267771403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,128,0,1,fp8,fp8,0,0.4122399886449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,128,0,1,float16,float16,0,0.387226661046346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,128,0,1,float16,fp8,0,0.3856906493504842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,128,0,1,float16,float16,0,0.4960586627324422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,128,0,1,fp8,fp8,0,0.32734400033950806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,128,0,1,float16,fp8,0,0.3883519967397054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,128,0,1,fp8,fp8,0,0.3346453507741292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,128,0,1,float16,fp8,0,0.4806026617685954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,128,0,1,fp8,fp8,0,0.3319200078646342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,128,0,1,float16,float16,0,0.39160533746083576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,128,0,1,float16,fp8,0,0.3888266483942668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,128,0,1,fp8,fp8,0,0.3423733313878377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,128,0,1,float16,fp8,0,0.25310399134953815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,128,0,1,fp8,fp8,0,0.2157813310623169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,128,0,1,float16,fp8,0,0.20414400100708008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,128,0,1,fp8,fp8,0,0.15948266784350076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,128,0,1,float16,float16,0,0.3866506814956665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,128,0,1,float16,float16,0,0.20644267400105795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,128,0,1,float16,float16,0,0.3808799982070923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,128,0,1,float16,fp8,0,0.20664532979329428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,128,0,1,fp8,fp8,0,0.15944533546765646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,128,0,1,float16,fp8,0,0.3831253449122111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,128,0,1,float16,float16,0,0.20571200052897134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,128,0,1,fp8,fp8,0,0.16098666191101074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,128,0,1,float16,fp8,0,0.20382400353749594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,128,0,1,float16,float16,0,0.20557334025700888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,128,0,1,float16,float16,0,0.2595946590105693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,128,0,1,float16,float16,0,0.20146665970484415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,128,0,1,float16,fp8,0,0.2063360015551249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,128,0,1,fp8,fp8,0,0.16317333777745566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,128,0,1,float16,float16,0,0.15152000387509665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,128,0,1,fp8,fp8,0,0.11223466197649638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,128,0,1,float16,float16,0,0.10760533809661865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,128,0,1,float16,fp8,0,0.10727999607721965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,128,0,1,float16,fp8,0,0.1476800044377645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,128,0,1,fp8,fp8,0,0.08690133690834045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,128,0,1,float16,float16,0,0.10938666264216106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,128,0,1,float16,fp8,0,0.10931199789047241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,128,0,1,fp8,fp8,0,0.08880533774693807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,128,0,1,float16,float16,0,0.10781866312026978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,128,0,1,float16,fp8,0,0.10842667023340861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,128,0,1,fp8,fp8,0,0.08805333574612935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,128,0,1,float16,float16,0,0.10910399754842122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,128,0,1,float16,fp8,0,0.11085866888364156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,128,0,1,fp8,fp8,0,0.09105066458384196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,128,0,1,fp8,fp8,0,8.232762654622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,128,0,1,fp8,fp8,0,8.301600138346354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,128,0,1,fp8,fp8,0,8.305248260498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,128,0,1,fp8,fp8,0,8.742245356241861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,128,0,1,float16,float16,0,6.385077158610026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,128,0,1,float16,fp8,0,6.167952219645183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,128,0,1,float16,float16,0,4.476058642069499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,128,0,1,float16,fp8,0,4.506064097086589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,128,0,1,fp8,fp8,0,4.128431955973308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,128,0,1,float16,float16,0,4.476863861083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,128,0,1,float16,fp8,0,4.513717333475749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,128,0,1,fp8,fp8,0,4.12334410349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,128,0,1,float16,float16,0,4.547562599182129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,128,0,1,float16,fp8,0,4.49454402923584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,128,0,1,fp8,fp8,0,4.125376065572103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,128,0,1,float16,float16,0,4.53659725189209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,128,0,1,float16,fp8,0,4.592981338500977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,128,0,1,fp8,fp8,0,4.361839930216472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,128,0,1,float16,fp8,0,3.110368092854818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,128,0,1,fp8,fp8,0,2.6815573374430337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,128,0,1,fp8,fp8,0,2.087615966796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,128,0,1,float16,fp8,0,2.257904052734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,128,0,1,float16,float16,0,3.2083307902018228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,128,0,1,fp8,fp8,0,2.0912906328837075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,128,0,1,float16,float16,0,2.2674612998962402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,128,0,1,float16,float16,0,2.2635253270467124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,128,0,1,float16,fp8,0,2.2493653297424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,128,0,1,float16,float16,0,2.2547252972920737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,128,0,1,float16,fp8,0,2.2543999354044595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,128,0,1,float16,float16,0,2.277045408884684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,128,0,1,float16,fp8,0,1.547200043996175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,128,0,1,fp8,fp8,0,2.1639092763264975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,128,0,1,fp8,fp8,0,2.0716212590535483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,128,0,1,fp8,fp8,0,1.3382986386617024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,128,0,1,float16,float16,0,1.5998293558756511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,128,0,1,float16,fp8,0,2.2714079221089682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,128,0,1,float16,float16,0,1.1541333198547363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,128,0,1,float16,fp8,0,1.1398133436838787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,128,0,1,fp8,fp8,0,1.0271680355072021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,128,0,1,float16,float16,0,1.132047971089681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,128,0,1,fp8,fp8,0,1.0225813388824463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,128,0,1,float16,float16,0,1.1561813354492188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,128,0,1,float16,fp8,0,1.143503983815511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,128,0,1,fp8,fp8,0,1.0451306502024333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,128,0,1,float16,float16,0,1.1569173336029053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,128,0,1,float16,fp8,0,1.1459519863128662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,128,0,1,float16,fp8,0,1.1340586344401042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,128,0,1,float16,fp8,0,0.7734026908874512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,128,0,1,fp8,fp8,0,1.11900798479716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,128,0,1,fp8,fp8,0,0.6785066922505697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,128,0,1,float16,float16,0,0.5833760102589926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,128,0,1,float16,fp8,0,0.5785173177719116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,128,0,1,fp8,fp8,0,0.5140159924825033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,128,0,1,float16,float16,0,0.5897973378499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,128,0,1,float16,fp8,0,0.5824906826019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,128,0,1,float16,float16,0,0.8086400032043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,128,0,1,fp8,fp8,0,0.5200479825337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,128,0,1,float16,float16,0,0.5832053422927856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,128,0,1,float16,fp8,0,0.5824000040690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,128,0,1,fp8,fp8,0,0.5332693258921305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,128,0,1,float16,float16,0,0.5897333224614462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,128,0,1,float16,fp8,0,0.5883520046869913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,128,0,1,fp8,fp8,0,0.5586773157119751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,128,0,1,float16,float16,0,0.4153866767883301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,128,0,1,float16,fp8,0,0.3991306622823079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,128,0,1,fp8,fp8,0,0.34884798526763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,128,0,1,float16,float16,0,0.3017279903093974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,128,0,1,fp8,fp8,0,0.2666026751200358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,128,0,1,float16,fp8,0,0.3035946687062581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,128,0,1,float16,float16,0,0.30291199684143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,128,0,1,float16,float16,0,0.3030719955762227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,128,0,1,float16,fp8,0,0.30288533369700116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,128,0,1,fp8,fp8,0,0.27083732684453327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,128,0,1,float16,fp8,0,0.3034293254216512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,128,0,1,float16,float16,0,0.3067733248074849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,128,0,1,float16,fp8,0,0.3062079946200053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,128,0,1,fp8,fp8,0,0.2831466595331828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,128,0,1,fp8,fp8,0,0.2710240085919698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,128,0,1,float16,float16,0,0.218831996122996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,128,0,1,float16,float16,0,0.15928000211715698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,128,0,1,fp8,fp8,0,0.18522133429845175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,128,0,1,fp8,fp8,0,0.1322986682256063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,128,0,1,float16,fp8,0,0.15963199734687805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,128,0,1,float16,float16,0,0.16198933124542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,128,0,1,float16,fp8,0,0.16209600369135538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,128,0,1,float16,fp8,0,0.2113920052846273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,128,0,1,fp8,fp8,0,0.13260266184806824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,128,0,1,float16,float16,0,0.16190399726231894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,128,0,1,float16,fp8,0,0.161189337571462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,128,0,1,fp8,fp8,0,0.13573333621025085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,128,0,1,float16,float16,0,0.16239466269810995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,128,0,1,float16,fp8,0,0.16397333145141602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,128,0,1,fp8,fp8,0,0.13666133085886636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,128,0,1,float16,float16,0,0.08703999718030293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,128,0,1,float16,float16,0,0.13341333468755087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,128,0,1,float16,fp8,0,0.13077867031097412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,128,0,1,fp8,fp8,0,0.0731573353211085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,128,0,1,fp8,fp8,0,0.09737066427866618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,128,0,1,float16,fp8,0,0.08749333024024963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,128,0,1,float16,float16,0,0.08710400263468425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,128,0,1,float16,fp8,0,0.08819199601809184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,128,0,1,fp8,fp8,0,0.0734506646792094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,128,0,1,float16,float16,0,0.0885706643263499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,128,0,1,float16,fp8,0,0.08807999889055888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,128,0,1,fp8,fp8,0,0.07439466814200084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,128,0,1,float16,float16,0,0.08923199772834778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,128,0,1,float16,float16,0,0.06549866497516632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,128,0,1,float16,fp8,0,0.08934932947158813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,128,0,1,fp8,fp8,0,0.07715199887752533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,128,0,1,float16,float16,0,0.04983466863632202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,128,0,1,float16,fp8,0,0.0633653352657954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,128,0,1,float16,fp8,0,0.04877333343029022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,128,0,1,fp8,fp8,0,0.05690666536490122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,128,0,1,fp8,fp8,0,0.042949333786964417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,128,0,1,float16,float16,0,0.04990399877230326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,128,0,1,float16,fp8,0,0.04944533109664917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,128,0,1,fp8,fp8,0,0.042133331298828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,128,0,1,float16,float16,0,0.04930666585763296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,128,0,1,float16,fp8,0,0.05037866532802582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,128,0,1,fp8,fp8,0,0.043162668744723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,128,0,1,float16,float16,0,0.0505973349014918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,128,0,1,float16,fp8,0,0.04975999891757965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,128,0,1,fp8,fp8,0,0.0446720023949941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,128,0,1,fp8,fp8,0,3.321455955505371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,128,0,1,float16,float16,0,3.8789278666178384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,128,0,1,float16,fp8,0,3.8530667622884116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,128,0,1,float16,float16,0,3.867120107014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,128,0,1,float16,fp8,0,3.893434524536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,128,0,1,fp8,fp8,0,3.277637481689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,128,0,1,fp8,fp8,0,3.3169867197672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,128,0,1,float16,float16,0,3.888784090677897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,128,0,1,float16,fp8,0,3.9046506881713867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,128,0,1,float16,float16,0,4.008981386820476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,128,0,1,float16,fp8,0,3.960511843363444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,128,0,1,float16,float16,0,1.9353599548339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,128,0,1,fp8,fp8,0,2.315397262573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,128,0,1,fp8,fp8,0,3.551520029703776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,128,0,1,float16,float16,0,2.9532852172851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,128,0,1,float16,fp8,0,2.836202621459961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,128,0,1,fp8,fp8,0,1.6327039400736492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,128,0,1,float16,fp8,0,1.930239995320638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,128,0,1,float16,float16,0,1.934666633605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,128,0,1,float16,fp8,0,1.9356533686319988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,128,0,1,fp8,fp8,0,1.6545119285583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,128,0,1,float16,float16,0,1.941658655802409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,128,0,1,fp8,fp8,0,1.6466986338297527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,128,0,1,float16,fp8,0,1.9572960535685222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,128,0,1,float16,float16,0,1.4842774073282878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,128,0,1,fp8,fp8,0,1.7930612564086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,128,0,1,float16,float16,0,1.9803999265034993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,128,0,1,float16,fp8,0,1.9598612785339355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,128,0,1,fp8,fp8,0,1.1632906595865886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,128,0,1,fp8,fp8,0,0.8295839627583822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,128,0,1,float16,float16,0,0.9759093125661215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,128,0,1,float16,fp8,0,1.4182933171590169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,128,0,1,float16,fp8,0,0.9848159948984782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,128,0,1,float16,float16,0,0.9792160193125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,128,0,1,fp8,fp8,0,0.8376053174336752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,128,0,1,float16,fp8,0,0.9885813395182291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,128,0,1,float16,float16,0,0.9812266826629639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,128,0,1,fp8,fp8,0,0.8370239734649658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,128,0,1,float16,fp8,0,0.9798719882965088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,128,0,1,float16,float16,0,0.9922453562418619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,128,0,1,float16,float16,0,0.7441653410593668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,128,0,1,fp8,fp8,0,0.9070666631062826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,128,0,1,float16,fp8,0,1.0001440048217773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,128,0,1,fp8,fp8,0,0.5808746814727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,128,0,1,float16,fp8,0,0.7156960169474283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,128,0,1,float16,float16,0,0.49910934766133624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,128,0,1,float16,fp8,0,0.49797332286834717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,128,0,1,fp8,fp8,0,0.4099200169245402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,128,0,1,float16,float16,0,0.5060906807581583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,128,0,1,float16,fp8,0,0.49772799015045166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,128,0,1,float16,float16,0,0.5003946622212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,128,0,1,fp8,fp8,0,0.4163893461227417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,128,0,1,float16,fp8,0,0.5013866821924845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,128,0,1,fp8,fp8,0,0.4220000108083089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,128,0,1,float16,float16,0,0.5106186469395956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,128,0,1,float16,fp8,0,0.5068213144938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,128,0,1,float16,float16,0,0.38254932562510174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,128,0,1,float16,fp8,0,0.36768531799316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,128,0,1,fp8,fp8,0,0.4453599850336711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,128,0,1,fp8,fp8,0,0.29959466060002643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,128,0,1,float16,float16,0,0.26230400800704956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,128,0,1,float16,fp8,0,0.25968533754348755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,128,0,1,fp8,fp8,0,0.21409600973129272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,128,0,1,float16,float16,0,0.2584693431854248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,128,0,1,float16,fp8,0,0.2613919973373413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,128,0,1,fp8,fp8,0,0.21896000703175864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,128,0,1,float16,float16,0,0.26318933566411334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,128,0,1,float16,fp8,0,0.26146666208902997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,128,0,1,fp8,fp8,0,0.21965867280960083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,128,0,1,float16,float16,0,0.2632799943288167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,128,0,1,float16,fp8,0,0.2624800006548564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,128,0,1,fp8,fp8,0,0.2340373396873474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,128,0,1,float16,float16,0,0.2027733325958252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,128,0,1,float16,fp8,0,0.19530133406321207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,128,0,1,fp8,fp8,0,0.1697280009587606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,128,0,1,float16,float16,0,0.13668800393740335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,128,0,1,float16,fp8,0,0.13886933525403342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,128,0,1,fp8,fp8,0,0.1181013286113739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,128,0,1,float16,float16,0,0.13798399766286215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,128,0,1,float16,float16,0,0.14096533258756003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,128,0,1,float16,fp8,0,0.14031466841697693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,128,0,1,fp8,fp8,0,0.11698666214942932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,128,0,1,float16,fp8,0,0.14231999715169272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,128,0,1,fp8,fp8,0,0.11737599968910217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,128,0,1,float16,float16,0,0.14198933045069376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,128,0,1,float16,fp8,0,0.14298666516939798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,128,0,1,fp8,fp8,0,0.12319466471672058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,128,0,1,float16,float16,0,0.12268799543380737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,128,0,1,float16,fp8,0,0.1200373371442159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,128,0,1,fp8,fp8,0,0.09284800291061401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,128,0,1,float16,float16,0,0.07764799892902374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,128,0,1,float16,fp8,0,0.07858666777610779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,128,0,1,fp8,fp8,0,0.06497600177923839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,128,0,1,float16,float16,0,0.07732800145943959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,128,0,1,float16,fp8,0,0.07870399951934814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,128,0,1,fp8,fp8,0,0.0656160016854604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,128,0,1,float16,float16,0,0.07885333398977916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,128,0,1,float16,fp8,0,0.0787306676308314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,128,0,1,fp8,fp8,0,0.06636266907056172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,128,0,1,float16,float16,0,0.08005333443482716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,128,0,1,float16,fp8,0,0.08011200030644734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,128,0,1,fp8,fp8,0,0.06912533442179362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,128,0,1,float16,float16,0,0.06301866471767426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,128,0,1,float16,fp8,0,0.060458665092786155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,128,0,1,float16,fp8,0,0.04456000030040741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,128,0,1,fp8,fp8,0,0.05301866432030996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,128,0,1,float16,float16,0,0.04429866870244344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,128,0,1,fp8,fp8,0,0.03842133283615112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,128,0,1,float16,float16,0,0.04399466514587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,128,0,1,float16,fp8,0,0.0440533310174942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,128,0,1,fp8,fp8,0,0.03738666574160258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,128,0,1,float16,float16,0,0.04460266729195913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,128,0,1,float16,fp8,0,0.04491200049718221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,128,0,1,fp8,fp8,0,0.038047999143600464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,128,0,1,float16,float16,0,0.045050665736198425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,128,0,1,float16,fp8,0,0.04524800181388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,128,0,1,fp8,fp8,0,0.039461334546407066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,128,0,1,float16,float16,0,0.03505066782236099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,128,0,1,float16,fp8,0,0.03426666557788849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,128,0,1,fp8,fp8,0,0.033301333586374916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,128,0,1,float16,float16,0,0.03136000037193298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,128,0,1,float16,fp8,0,0.031770666440327965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,128,0,1,fp8,fp8,0,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,128,0,1,float16,float16,0,0.03179199993610382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,128,0,1,float16,float16,0,0.03129599988460541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,128,0,1,float16,fp8,0,0.03254399945338567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,128,0,1,fp8,fp8,0,0.027717334528764088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,128,0,1,float16,fp8,0,0.03165333221356074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,128,0,1,fp8,fp8,0,0.028394666810830433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,128,0,1,float16,float16,0,0.03158933420976003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,128,0,1,float16,fp8,0,0.032357332607110344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,128,0,1,fp8,fp8,0,0.028362666567166645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,128,0,1,float16,float16,0,1.7981546719868977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,128,0,1,fp8,fp8,0,1.619642734527588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,128,0,1,float16,fp8,0,1.80076265335083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,128,0,1,float16,float16,0,1.8055893580118816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,128,0,1,float16,fp8,0,1.804426670074463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,128,0,1,fp8,fp8,0,1.6334667205810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,128,0,1,float16,float16,0,1.8041973114013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,128,0,1,fp8,fp8,0,1.6392107009887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,128,0,1,float16,fp8,0,1.8033332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,128,0,1,float16,fp8,0,1.8500693639119465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,128,0,1,float16,float16,0,1.8467413584391277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,128,0,1,fp8,fp8,0,1.7830026944478352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,128,0,1,fp8,fp8,0,1.1625973383585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,128,0,1,float16,float16,0,1.5161867141723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,128,0,1,float16,fp8,0,1.4580693244934082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,128,0,1,float16,float16,0,0.9116426308949789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,128,0,1,float16,fp8,0,0.9077706336975098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,128,0,1,fp8,fp8,0,0.808351993560791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,128,0,1,float16,float16,0,0.9125813643137614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,128,0,1,fp8,fp8,0,0.811247984568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,128,0,1,float16,fp8,0,0.9111839930216471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,128,0,1,float16,float16,0,0.91429336865743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,128,0,1,float16,fp8,0,0.912010669708252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,128,0,1,fp8,fp8,0,0.8378187020619711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,128,0,1,float16,float16,0,0.929375966389974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,128,0,1,float16,fp8,0,0.9247999986012777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,128,0,1,fp8,fp8,0,0.9006400108337402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,128,0,1,float16,float16,0,0.7669599850972494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,128,0,1,float16,fp8,0,0.7319466272989908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,128,0,1,fp8,fp8,0,0.5838720003763834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,128,0,1,float16,float16,0,0.465776006380717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,128,0,1,float16,fp8,0,0.4672853151957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,128,0,1,fp8,fp8,0,0.4080959955851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,128,0,1,float16,float16,0,0.4652213255564372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,128,0,1,float16,fp8,0,0.4647253354390462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,128,0,1,fp8,fp8,0,0.411082665125529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,128,0,1,float16,float16,0,0.46566398938496906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,128,0,1,float16,fp8,0,0.4672000010808309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,128,0,1,fp8,fp8,0,0.42022931575775146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,128,0,1,float16,float16,0,0.475818673769633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,128,0,1,float16,fp8,0,0.4723413387934367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,128,0,1,fp8,fp8,0,0.4527519941329956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,128,0,1,float16,float16,0,0.3896799882253011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,128,0,1,float16,fp8,0,0.37428800264994305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,128,0,1,fp8,fp8,0,0.2949120004971822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,128,0,1,float16,float16,0,0.24238399664560953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,128,0,1,float16,fp8,0,0.24293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,128,0,1,fp8,fp8,0,0.21060800552368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,128,0,1,float16,float16,0,0.24339733521143594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,128,0,1,float16,fp8,0,0.2440426747004191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,128,0,1,float16,float16,0,0.2453493277231852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,128,0,1,fp8,fp8,0,0.2102880080540975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,128,0,1,float16,fp8,0,0.24528000752131143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,128,0,1,fp8,fp8,0,0.216922660668691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,128,0,1,float16,float16,0,0.24962133169174194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,128,0,1,float16,fp8,0,0.2501866618792216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,128,0,1,fp8,fp8,0,0.22572267055511475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,128,0,1,float16,float16,0,0.20786666870117188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,128,0,1,float16,fp8,0,0.20126932859420776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,128,0,1,fp8,fp8,0,0.1585813363393148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,128,0,1,float16,float16,0,0.127920001745224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,128,0,1,float16,fp8,0,0.12796800335248312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,128,0,1,fp8,fp8,0,0.11483200391133626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,128,0,1,float16,float16,0,0.12964266538619995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,128,0,1,float16,fp8,0,0.13063466548919678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,128,0,1,fp8,fp8,0,0.11675199866294861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,128,0,1,float16,float16,0,0.1304426689942678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,128,0,1,float16,fp8,0,0.13191466530164084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,128,0,1,fp8,fp8,0,0.11905066172281902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,128,0,1,float16,float16,0,0.133925328652064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,128,0,1,float16,fp8,0,0.1325386663277944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,128,0,1,fp8,fp8,0,0.11875200271606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,128,0,1,float16,fp8,0,0.11319466431935628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,128,0,1,float16,float16,0,0.1160426636536916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,128,0,1,fp8,fp8,0,0.08941333492596944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,128,0,1,float16,float16,0,0.07338133454322815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,128,0,1,float16,fp8,0,0.0746613343556722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,128,0,1,fp8,fp8,0,0.06462400158246358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,128,0,1,fp8,fp8,0,0.06554666658242543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,128,0,1,float16,float16,0,0.07358400026957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,128,0,1,float16,fp8,0,0.07394133508205414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,128,0,1,float16,float16,0,0.07529599964618683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,128,0,1,float16,fp8,0,0.07462400197982788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,128,0,1,fp8,fp8,0,0.0661653329928716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,128,0,1,fp8,fp8,0,0.0687360018491745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,128,0,1,float16,float16,0,0.07542400062084198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,128,0,1,float16,fp8,0,0.07553599774837494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,128,0,1,float16,float16,0,0.0625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,128,0,1,float16,fp8,0,0.059818665186564125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,128,0,1,float16,fp8,0,0.04266666869322459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,128,0,1,fp8,fp8,0,0.053354665637016296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,128,0,1,float16,float16,0,0.041802664597829185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,128,0,1,float16,float16,0,0.04262933135032654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,128,0,1,fp8,fp8,0,0.03664000084002813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,128,0,1,float16,float16,0,0.04253333310286204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,128,0,1,float16,fp8,0,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,128,0,1,fp8,fp8,0,0.037274666130542755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,128,0,1,float16,fp8,0,0.04301333427429199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,128,0,1,fp8,fp8,0,0.03703466554482778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,128,0,1,float16,float16,0,0.04316799839337667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,128,0,1,float16,fp8,0,0.043525333205858864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,128,0,1,fp8,fp8,0,0.039359999199708305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,128,0,1,fp8,fp8,0,0.03285333265860876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,128,0,1,float16,float16,0,0.034287999073664345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,128,0,1,float16,fp8,0,0.03390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,128,0,1,float16,float16,0,0.031317333380381264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,128,0,1,float16,fp8,0,0.031930667658646904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,128,0,1,fp8,fp8,0,0.027989332874615986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,128,0,1,float16,float16,0,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,128,0,1,float16,float16,0,0.031888000667095184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,128,0,1,float16,fp8,0,0.031221332649389904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,128,0,1,fp8,fp8,0,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,128,0,1,float16,fp8,0,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,128,0,1,fp8,fp8,0,0.02762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,128,0,1,float16,float16,0,0.03187733391920725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,128,0,1,float16,fp8,0,0.031744000812371574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,128,0,1,fp8,fp8,0,0.027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,128,0,1,float16,float16,0,0.02214933435122172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,128,0,1,float16,fp8,0,0.02309333284695943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,128,0,1,fp8,fp8,0,0.022495999932289124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,128,0,1,float16,float16,0,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,128,0,1,float16,fp8,0,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,128,0,1,fp8,fp8,0,0.02015999952952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,128,0,1,float16,float16,0,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,128,0,1,float16,fp8,0,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,128,0,1,fp8,fp8,0,0.02042666698495547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,128,0,1,float16,float16,0,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,128,0,1,float16,fp8,0,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,128,0,1,fp8,fp8,0,0.020362666497627895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,128,0,1,float16,float16,0,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,128,0,1,float16,fp8,0,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,128,0,1,fp8,fp8,0,0.02083733429511388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,128,0,1,float16,float16,0,1.050005356470744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,128,0,1,float16,fp8,0,1.0488639672597249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,128,0,1,float16,float16,0,1.052074670791626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,128,0,1,fp8,fp8,0,1.1522719860076904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,128,0,1,float16,fp8,0,1.0590559641520183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,128,0,1,fp8,fp8,0,1.1424213250478108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,128,0,1,float16,float16,0,1.0548693339029949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,128,0,1,float16,fp8,0,1.0529226462046306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,128,0,1,fp8,fp8,0,1.1508639653523762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,128,0,1,float16,float16,0,1.0790186723073323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,128,0,1,float16,fp8,0,1.0661760171254475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,128,0,1,fp8,fp8,0,1.2338666915893555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,128,0,1,float16,float16,0,0.7689493497212728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,128,0,1,float16,fp8,0,0.739898681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,128,0,1,fp8,fp8,0,0.7461493015289307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,128,0,1,float16,float16,0,0.5320320129394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,128,0,1,float16,fp8,0,0.5312960147857666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,128,0,1,fp8,fp8,0,0.5673706531524658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,128,0,1,float16,float16,0,0.5308320124944051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,128,0,1,float16,fp8,0,0.5321813424428304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,128,0,1,fp8,fp8,0,0.5723199844360352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,128,0,1,float16,float16,0,0.5332693258921305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,128,0,1,float16,fp8,0,0.53657066822052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,128,0,1,fp8,fp8,0,0.5818560123443604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,128,0,1,float16,float16,0,0.5470879872639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,128,0,1,float16,fp8,0,0.5450453360875448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,128,0,1,fp8,fp8,0,0.6092640161514282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,128,0,1,float16,float16,0,0.39262934525807697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,128,0,1,float16,fp8,0,0.37836265563964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,128,0,1,fp8,fp8,0,0.3782293399175008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,128,0,1,float16,float16,0,0.2767626643180847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,128,0,1,float16,fp8,0,0.27403199672698975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,128,0,1,fp8,fp8,0,0.2962239980697632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,128,0,1,float16,float16,0,0.27435733874638873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,128,0,1,float16,fp8,0,0.27484800418217975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,128,0,1,fp8,fp8,0,0.30037333567937213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,128,0,1,float16,float16,0,0.2746346592903137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,128,0,1,float16,fp8,0,0.2756906747817993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,128,0,1,fp8,fp8,0,0.3003786603609721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,128,0,1,float16,float16,0,0.2802986701329549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,128,0,1,float16,float16,0,0.20639467239379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,128,0,1,float16,fp8,0,0.1983733375867208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,128,0,1,fp8,fp8,0,0.19946666558583578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,128,0,1,float16,fp8,0,0.2823626597722371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,128,0,1,fp8,fp8,0,0.3102026581764221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,128,0,1,float16,float16,0,0.14140799641609192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,128,0,1,float16,fp8,0,0.14170666535695395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,128,0,1,fp8,fp8,0,0.1577173372109731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,128,0,1,float16,float16,0,0.14380266269048056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,128,0,1,float16,fp8,0,0.143696000178655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,128,0,1,fp8,fp8,0,0.15922666589419046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,128,0,1,fp8,fp8,0,0.16006400187810263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,128,0,1,float16,float16,0,0.145989328622818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,128,0,1,float16,fp8,0,0.14410666624704996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,128,0,1,fp8,fp8,0,0.1637386679649353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,128,0,1,float16,float16,0,0.14735999703407288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,128,0,1,float16,fp8,0,0.14782933394114176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,128,0,1,float16,float16,0,0.11306132872899373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,128,0,1,float16,fp8,0,0.10971732934315999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,128,0,1,float16,float16,0,0.07681066791216533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,128,0,1,fp8,fp8,0,0.10869333148002625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,128,0,1,float16,fp8,0,0.07765333354473114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,128,0,1,fp8,fp8,0,0.08760533730189006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,128,0,1,float16,fp8,0,0.07843199868996938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,128,0,1,float16,fp8,0,0.07938133180141449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,128,0,1,float16,float16,0,0.0783679982026418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,128,0,1,fp8,fp8,0,0.08724799752235413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,128,0,1,float16,float16,0,0.07828799883524577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,128,0,1,fp8,fp8,0,0.08849599957466125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,128,0,1,fp8,fp8,0,0.0904960036277771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,128,0,1,float16,float16,0,0.07860800127188365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,128,0,1,float16,fp8,0,0.08021866778532664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,128,0,1,float16,float16,0,0.06283199787139893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,128,0,1,float16,fp8,0,0.060959999759991966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,128,0,1,fp8,fp8,0,0.06432533264160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,128,0,1,float16,float16,0,0.0447573314110438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,128,0,1,float16,fp8,0,0.044495999813079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,128,0,1,fp8,fp8,0,0.04927466809749603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,128,0,1,float16,float16,0,0.04468800127506256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,128,0,1,float16,fp8,0,0.044719999035199486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,128,0,1,fp8,fp8,0,0.04939733445644379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,128,0,1,fp8,fp8,0,0.049045334259668984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,128,0,1,float16,float16,0,0.04489600161711375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,128,0,1,float16,fp8,0,0.04469866553942362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,128,0,1,float16,float16,0,0.045706664522488914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,128,0,1,float16,fp8,0,0.04647466540336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,128,0,1,fp8,fp8,0,0.05073066552480062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,128,0,1,float16,float16,0,0.03348266581694285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,128,0,1,float16,float16,0,0.030074665943781536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,128,0,1,float16,fp8,0,0.032885332902272545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,128,0,1,fp8,fp8,0,0.03706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,128,0,1,float16,fp8,0,0.03065599997838338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,128,0,1,fp8,fp8,0,0.03181333343187968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,128,0,1,float16,float16,0,0.030181333422660828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,128,0,1,float16,float16,0,0.03050133337577184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,128,0,1,float16,fp8,0,0.029461334149042766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,128,0,1,fp8,fp8,0,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,128,0,1,float16,fp8,0,0.02972800036271413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,128,0,1,fp8,fp8,0,0.03158933420976003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,128,0,1,float16,float16,0,0.030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,128,0,1,float16,fp8,0,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,128,0,1,fp8,fp8,0,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,128,0,1,float16,float16,0,0.02421333392461141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,128,0,1,float16,fp8,0,0.023818666736284893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,128,0,1,fp8,fp8,0,0.023397333920001984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,128,0,1,fp8,fp8,0,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,128,0,1,float16,float16,0,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,128,0,1,float16,fp8,0,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,128,0,1,float16,float16,0,0.022800001005331676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,128,0,1,float16,fp8,0,0.022645334402720135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,128,0,1,fp8,fp8,0,0.02378133436044057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,128,0,1,float16,fp8,0,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,128,0,1,float16,float16,0,0.022698665658632915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,128,0,1,fp8,fp8,0,0.024256000916163128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,128,0,1,float16,float16,0,0.022495999932289124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,128,0,1,float16,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,128,0,1,float16,float16,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,128,0,1,fp8,fp8,0,0.024256000916163128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,128,0,1,float16,fp8,0,0.022831998765468597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,128,0,1,fp8,fp8,0,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,128,0,1,float16,float16,0,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,128,0,1,float16,fp8,0,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,128,0,1,fp8,fp8,0,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,128,0,1,fp8,fp8,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,128,0,1,float16,float16,0,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,128,0,1,float16,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,128,0,1,float16,float16,0,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,128,0,1,float16,fp8,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,128,0,1,fp8,fp8,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,128,0,1,fp8,fp8,0,0.01781333362062772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,128,0,1,float16,float16,0,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,128,0,1,float16,fp8,0,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,128,0,1,float16,float16,0,0.7231893539428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,128,0,1,float16,fp8,0,0.7219839890797933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,128,0,1,float16,float16,0,0.7197653452555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,128,0,1,fp8,fp8,0,0.8947679996490479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,128,0,1,float16,fp8,0,0.720741351445516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,128,0,1,fp8,fp8,0,0.909231980641683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,128,0,1,float16,float16,0,0.7273279825846354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,128,0,1,float16,fp8,0,0.7241013050079346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,128,0,1,float16,float16,0,0.7319040298461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,128,0,1,fp8,fp8,0,0.9053920110066732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,128,0,1,float16,float16,0,0.5014346837997437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,128,0,1,float16,fp8,0,0.7348053455352783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,128,0,1,float16,fp8,0,0.4843786557515462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,128,0,1,fp8,fp8,0,0.5358773469924927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,128,0,1,fp8,fp8,0,0.9508319695790609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,128,0,1,float16,float16,0,0.36686933040618896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,128,0,1,float16,fp8,0,0.36745599905649823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,128,0,1,fp8,fp8,0,0.45660801728566486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,128,0,1,float16,float16,0,0.3692266543706258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,128,0,1,float16,fp8,0,0.370250662167867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,128,0,1,fp8,fp8,0,0.4591573476791382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,128,0,1,float16,float16,0,0.3695146640141805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,128,0,1,float16,fp8,0,0.36857601006825763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,128,0,1,float16,float16,0,0.3806026776631673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,128,0,1,fp8,fp8,0,0.4650239944458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,128,0,1,float16,fp8,0,0.3750346501668294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,128,0,1,fp8,fp8,0,0.47275201479593915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,128,0,1,float16,float16,0,0.2555946707725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,128,0,1,float16,fp8,0,0.2508959968884786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,128,0,1,fp8,fp8,0,0.2779253323872884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,128,0,1,float16,float16,0,0.18690133094787598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,128,0,1,float16,fp8,0,0.1855306625366211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,128,0,1,fp8,fp8,0,0.24016533295313516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,128,0,1,float16,float16,0,0.19121599197387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,128,0,1,float16,fp8,0,0.19174933433532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,128,0,1,float16,float16,0,0.18981866041819254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,128,0,1,float16,fp8,0,0.19310933351516724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,128,0,1,fp8,fp8,0,0.24178133408228555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,128,0,1,fp8,fp8,0,0.24245333671569824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,128,0,1,float16,float16,0,0.19667200247446695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,128,0,1,float16,float16,0,0.13522133231163025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,128,0,1,float16,fp8,0,0.19682133197784424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,128,0,1,fp8,fp8,0,0.24207999308904013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,128,0,1,float16,float16,0,0.09987733761469524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,128,0,1,float16,fp8,0,0.13170666495958963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,128,0,1,fp8,fp8,0,0.1490079959233602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,128,0,1,float16,fp8,0,0.10097066561381023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,128,0,1,fp8,fp8,0,0.1283146639664968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,128,0,1,float16,float16,0,0.09939199686050415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,128,0,1,float16,fp8,0,0.10123200217882793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,128,0,1,fp8,fp8,0,0.12808000048001608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,128,0,1,float16,float16,0,0.10100799798965454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,128,0,1,float16,fp8,0,0.10242666800816853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,128,0,1,fp8,fp8,0,0.12980799873669943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,128,0,1,float16,float16,0,0.07436266541481018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,128,0,1,float16,float16,0,0.10054399569829305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,128,0,1,float16,fp8,0,0.10211199522018433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,128,0,1,fp8,fp8,0,0.13115200400352478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,128,0,1,float16,fp8,0,0.07230400045712788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,128,0,1,fp8,fp8,0,0.08402132987976074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,128,0,1,float16,float16,0,0.05514666438102722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,128,0,1,float16,fp8,0,0.055200000603993736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,128,0,1,fp8,fp8,0,0.06935999790827434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,128,0,1,float16,float16,0,0.054560000697771706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,128,0,1,float16,fp8,0,0.055488000313440956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,128,0,1,fp8,fp8,0,0.07046933472156525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,128,0,1,float16,float16,0,0.05492266515890757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,128,0,1,float16,fp8,0,0.05459199845790863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,128,0,1,fp8,fp8,0,0.06991999844710033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,128,0,1,float16,float16,0,0.05550399919350942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,128,0,1,float16,fp8,0,0.05675200124581655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,128,0,1,fp8,fp8,0,0.07152000069618225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,128,0,1,float16,float16,0,0.038831998904546104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,128,0,1,float16,fp8,0,0.038533332447210945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,128,0,1,fp8,fp8,0,0.047685335079828896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,128,0,1,float16,float16,0,0.03472533325354258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,128,0,1,float16,fp8,0,0.03454400102297465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,128,0,1,fp8,fp8,0,0.04279999931653341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,128,0,1,float16,float16,0,0.035216001172860466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,128,0,1,float16,float16,0,0.03489600121974945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,128,0,1,float16,fp8,0,0.03551999976237615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,128,0,1,fp8,fp8,0,0.04204266766707102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,128,0,1,float16,fp8,0,0.03508266558249792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,128,0,1,fp8,fp8,0,0.042992000778516136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,128,0,1,float16,float16,0,0.03565866748491923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,128,0,1,float16,fp8,0,0.035002666215101876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,128,0,1,fp8,fp8,0,0.043552001317342125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,128,0,1,float16,float16,0,0.02550933261712392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,128,0,1,float16,fp8,0,0.02606933315594991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,128,0,1,fp8,fp8,0,0.03046400099992752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,128,0,1,float16,float16,0,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,128,0,1,float16,fp8,0,0.023717333873112995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,128,0,1,fp8,fp8,0,0.027952000498771667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,128,0,1,float16,float16,0,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,128,0,1,float16,fp8,0,0.024288001159826916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,128,0,1,fp8,fp8,0,0.02826666583617528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,128,0,1,float16,float16,0,0.023792001108328503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,128,0,1,float16,fp8,0,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,128,0,1,fp8,fp8,0,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,128,0,1,float16,float16,0,0.02387733260790507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,128,0,1,float16,fp8,0,0.024522667129834492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,128,0,1,fp8,fp8,0,0.028650666276613872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,128,0,1,float16,float16,0,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,128,0,1,float16,fp8,0,0.019706666469573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,128,0,1,fp8,fp8,0,0.022282667458057404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,128,0,1,float16,float16,0,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,128,0,1,float16,fp8,0,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,128,0,1,fp8,fp8,0,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,128,0,1,float16,float16,0,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,128,0,1,float16,fp8,0,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,128,0,1,fp8,fp8,0,0.02182399978240331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,128,0,1,fp8,fp8,0,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,128,0,1,float16,float16,0,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,128,0,1,float16,fp8,0,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,128,0,1,float16,float16,0,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,128,0,1,float16,fp8,0,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,128,0,1,fp8,fp8,0,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,128,0,1,float16,float16,0,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,128,0,1,float16,fp8,0,0.01616000011563301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,128,0,1,fp8,fp8,0,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,128,0,1,float16,float16,0,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,128,0,1,float16,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,128,0,1,fp8,fp8,0,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,128,0,1,float16,float16,0,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,128,0,1,float16,fp8,0,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,128,0,1,fp8,fp8,0,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,128,0,1,float16,float16,0,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,128,0,1,float16,fp8,0,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,128,0,1,fp8,fp8,0,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,128,0,1,float16,float16,0,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,128,0,1,float16,fp8,0,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,128,0,1,fp8,fp8,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,128,0,1,float16,float16,0,0.5805919965108236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,128,0,1,float16,fp8,0,0.5776853164037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,128,0,1,float16,float16,0,0.585647980372111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,128,0,1,fp8,fp8,0,0.7951146761576334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,128,0,1,float16,fp8,0,0.5847786664962769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,128,0,1,float16,float16,0,0.5824533303578695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,128,0,1,fp8,fp8,0,0.7928640047709147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,128,0,1,float16,fp8,0,0.5856320063273112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,128,0,1,float16,float16,0,0.5937120119730631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,128,0,1,fp8,fp8,0,0.801088015238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,128,0,1,float16,float16,0,0.3893386522928874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,128,0,1,float16,fp8,0,0.590666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,128,0,1,fp8,fp8,0,0.8064373334248861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,128,0,1,float16,fp8,0,0.38602666060129803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,128,0,1,fp8,fp8,0,0.4461173216501872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,128,0,1,float16,float16,0,0.29418132702509564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,128,0,1,float16,fp8,0,0.2932479977607727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,128,0,1,fp8,fp8,0,0.40303464730580646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,128,0,1,float16,float16,0,0.29799999793370563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,128,0,1,float16,fp8,0,0.30003732442855835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,128,0,1,float16,float16,0,0.30074665943781537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,128,0,1,fp8,fp8,0,0.40907732645670575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,128,0,1,float16,fp8,0,0.3008906642595927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,128,0,1,fp8,fp8,0,0.4046666622161865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,128,0,1,float16,float16,0,0.30608532826105755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,128,0,1,float16,fp8,0,0.3057279984156291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,128,0,1,float16,float16,0,0.18578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,128,0,1,fp8,fp8,0,0.4084800084431966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,128,0,1,float16,fp8,0,0.18293867508570352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,128,0,1,fp8,fp8,0,0.22941333055496216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,128,0,1,float16,float16,0,0.15347199638684592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,128,0,1,float16,fp8,0,0.15195733308792114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,128,0,1,float16,float16,0,0.15317333738009134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,128,0,1,fp8,fp8,0,0.21024533112843832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,128,0,1,float16,float16,0,0.15481066703796387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,128,0,1,float16,fp8,0,0.1523413360118866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,128,0,1,fp8,fp8,0,0.20995734135309854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,128,0,1,fp8,fp8,0,0.2132479945818583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,128,0,1,float16,fp8,0,0.15402133266131082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,128,0,1,float16,float16,0,0.15350932876269022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,128,0,1,float16,fp8,0,0.15668800473213196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,128,0,1,float16,float16,0,0.09916800260543823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,128,0,1,fp8,fp8,0,0.21279466152191162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,128,0,1,float16,float16,0,0.08099199831485748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,128,0,1,float16,fp8,0,0.0981066624323527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,128,0,1,fp8,fp8,0,0.12618133425712585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,128,0,1,float16,fp8,0,0.0839573343594869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,128,0,1,fp8,fp8,0,0.11066133777300517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,128,0,1,float16,float16,0,0.08084266881148021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,128,0,1,float16,fp8,0,0.08274133503437042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,128,0,1,fp8,fp8,0,0.1125973363717397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,128,0,1,float16,float16,0,0.08182399968306224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,128,0,1,float16,fp8,0,0.0812799980243047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,128,0,1,fp8,fp8,0,0.11269866426785786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,128,0,1,float16,float16,0,0.08316799998283386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,128,0,1,float16,fp8,0,0.08381332953770955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,128,0,1,fp8,fp8,0,0.11220799883206685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,128,0,1,float16,float16,0,0.0517493337392807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,128,0,1,float16,fp8,0,0.053258667389551796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,128,0,1,fp8,fp8,0,0.0703359991312027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,128,0,1,float16,float16,0,0.0491946687301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,128,0,1,float16,fp8,0,0.04892266790072123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,128,0,1,fp8,fp8,0,0.064410666624705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,128,0,1,float16,float16,0,0.04885333279768626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,128,0,1,float16,fp8,0,0.049039999643961586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,128,0,1,fp8,fp8,0,0.06346666812896729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,128,0,1,float16,float16,0,0.04827733337879181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,128,0,1,float16,fp8,0,0.04807466765244802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,128,0,1,fp8,fp8,0,0.06445333361625671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,128,0,1,float16,float16,0,0.04814933240413666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,128,0,1,float16,float16,0,0.03313600023587545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,128,0,1,float16,fp8,0,0.04894933104515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,128,0,1,fp8,fp8,0,0.06453333298365276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,128,0,1,float16,fp8,0,0.0337119996547699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,128,0,1,fp8,fp8,0,0.041034666200478874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,128,0,1,float16,float16,0,0.03054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,128,0,1,float16,fp8,0,0.030159999926884968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,128,0,1,fp8,fp8,0,0.0390133336186409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,128,0,1,float16,float16,0,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,128,0,1,float16,fp8,0,0.031914666295051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,128,0,1,fp8,fp8,0,0.03834133346875509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,128,0,1,float16,float16,0,0.030576000610987347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,128,0,1,float16,fp8,0,0.03173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,128,0,1,fp8,fp8,0,0.0391893337170283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,128,0,1,float16,float16,0,0.03226666649182638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,128,0,1,float16,fp8,0,0.03179199993610382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,128,0,1,fp8,fp8,0,0.03909866760174433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,128,0,1,float16,float16,0,0.022730665902296703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,128,0,1,float16,fp8,0,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,128,0,1,fp8,fp8,0,0.026672000686327618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,128,0,1,float16,float16,0,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,128,0,1,float16,fp8,0,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,128,0,1,fp8,fp8,0,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,128,0,1,float16,float16,0,0.02242133269707362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,128,0,1,float16,fp8,0,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,128,0,1,fp8,fp8,0,0.025557334224383037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,128,0,1,float16,float16,0,0.022202665607134502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,128,0,1,float16,fp8,0,0.022357332209746044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,128,0,1,fp8,fp8,0,0.02622399975856145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,128,0,1,float16,float16,0,0.022250667214393616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,128,0,1,float16,fp8,0,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,128,0,1,fp8,fp8,0,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,128,0,1,float16,float16,0,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,128,0,1,float16,fp8,0,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,128,0,1,fp8,fp8,0,0.021967999637126923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,128,0,1,float16,float16,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,128,0,1,float16,fp8,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,128,0,1,fp8,fp8,0,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,128,0,1,float16,float16,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,128,0,1,float16,fp8,0,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,128,0,1,fp8,fp8,0,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,128,0,1,float16,float16,0,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,128,0,1,float16,fp8,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,128,0,1,fp8,fp8,0,0.021733333667119343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,128,0,1,fp8,fp8,0,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,128,0,1,float16,float16,0,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,128,0,1,float16,fp8,0,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,128,0,1,float16,float16,0,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,128,0,1,float16,fp8,0,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,128,0,1,fp8,fp8,0,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,128,0,1,float16,float16,0,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,128,0,1,float16,float16,0,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,128,0,1,float16,fp8,0,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,128,0,1,fp8,fp8,0,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,128,0,1,float16,fp8,0,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,128,0,1,fp8,fp8,0,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,128,0,1,float16,float16,0,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,128,0,1,float16,fp8,0,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,128,0,1,fp8,fp8,0,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,128,0,1,float16,float16,0,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,128,0,1,float16,fp8,0,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,128,0,1,fp8,fp8,0,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,128,0,1,fp8,fp8,0,47.3832753499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,128,0,1,fp8,fp8,0,49.40457661946615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,128,0,1,float16,fp8,0,65.21144612630208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,128,0,1,float16,float16,0,65.45261637369792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,128,0,1,float16,fp8,0,65.88940938313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,128,0,1,float16,float16,0,66.41822814941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,128,0,1,float16,float16,0,68.53833516438802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,128,0,1,fp8,fp8,0,46.57165018717448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,128,0,1,float16,fp8,0,68.76900227864583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,128,0,1,fp8,fp8,0,47.89229838053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,128,0,1,float16,float16,0,36.096778869628906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,128,0,1,float16,float16,0,66.23970031738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,128,0,1,float16,fp8,0,65.8232930501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,128,0,1,fp8,fp8,0,22.735738118489582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,128,0,1,float16,fp8,0,35.74115244547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,128,0,1,float16,float16,0,35.12588246663412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,128,0,1,float16,fp8,0,35.629259745279946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,128,0,1,fp8,fp8,0,21.854469299316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,128,0,1,fp8,fp8,0,23.024795532226562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,128,0,1,float16,fp8,0,33.21008046468099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,128,0,1,float16,float16,0,33.607434590657554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,128,0,1,fp8,fp8,0,22.730336507161457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,128,0,1,float16,float16,0,34.2754872639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,128,0,1,float16,fp8,0,34.86303456624349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,128,0,1,fp8,fp8,0,12.438084920247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,128,0,1,float16,float16,0,17.80898666381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,128,0,1,float16,float16,0,33.127296447753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,128,0,1,float16,fp8,0,16.904783884684246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,128,0,1,fp8,fp8,0,22.699567159016926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,128,0,1,float16,fp8,0,34.95706685384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,128,0,1,fp8,fp8,0,11.630570729573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,128,0,1,float16,float16,0,16.6920903523763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,128,0,1,float16,fp8,0,17.061908721923828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,128,0,1,fp8,fp8,0,11.160645802815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,128,0,1,float16,float16,0,17.317296346028645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,128,0,1,float16,fp8,0,17.88525390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,128,0,1,fp8,fp8,0,11.643658955891928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,128,0,1,float16,float16,0,16.84773890177409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,128,0,1,float16,fp8,0,16.82969029744466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,128,0,1,float16,float16,0,8.929781595865885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,128,0,1,float16,float16,0,17.715829213460285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,128,0,1,fp8,fp8,0,6.027088165283203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,128,0,1,float16,fp8,0,8.690591812133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,128,0,1,fp8,fp8,0,11.902458190917969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,128,0,1,float16,fp8,0,17.87881596883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,128,0,1,float16,float16,0,8.964832305908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,128,0,1,fp8,fp8,0,6.0491517384847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,128,0,1,float16,fp8,0,9.022810618082682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,128,0,1,fp8,fp8,0,5.593503952026367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,128,0,1,float16,float16,0,8.627706527709961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,128,0,1,float16,fp8,0,8.950560251871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,128,0,1,fp8,fp8,0,5.988885243733724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,128,0,1,float16,fp8,0,8.847904205322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,128,0,1,float16,float16,0,8.565098444620768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,128,0,1,fp8,fp8,0,5.897375742594401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,128,0,1,float16,float16,0,8.758384068806967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,128,0,1,float16,fp8,0,9.000656127929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,128,0,1,fp8,fp8,0,26.348495483398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,128,0,1,fp8,fp8,0,25.953114827473957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,128,0,1,float16,float16,0,38.77397918701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,128,0,1,float16,fp8,0,40.05786641438802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,128,0,1,float16,float16,0,39.81644185384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,128,0,1,float16,fp8,0,38.77212270100912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,128,0,1,float16,float16,0,39.375134785970054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,128,0,1,float16,fp8,0,40.00268809000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,128,0,1,fp8,fp8,0,26.08289082845052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,128,0,1,fp8,fp8,0,13.95623524983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,128,0,1,float16,float16,0,20.581456502278645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,128,0,1,float16,fp8,0,20.644784291585285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,128,0,1,float16,float16,0,19.731131235758465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,128,0,1,fp8,fp8,0,27.24171193440755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,128,0,1,float16,float16,0,38.80053456624349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,128,0,1,fp8,fp8,0,13.475770314534506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,128,0,1,float16,fp8,0,19.927162170410156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,128,0,1,float16,fp8,0,39.417765299479164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,128,0,1,fp8,fp8,0,13.532763163248697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,128,0,1,float16,float16,0,19.529978434244793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,128,0,1,float16,fp8,0,19.249844868977863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,128,0,1,fp8,fp8,0,13.326314290364584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,128,0,1,float16,float16,0,19.964922587076824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,128,0,1,float16,fp8,0,20.008778889973957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,128,0,1,fp8,fp8,0,13.288448333740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,128,0,1,float16,float16,0,10.245957056681315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,128,0,1,float16,fp8,0,9.874757130940756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,128,0,1,float16,float16,0,19.52998987833659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,128,0,1,fp8,fp8,0,7.235205332438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,128,0,1,float16,fp8,0,20.066394805908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,128,0,1,float16,float16,0,10.027754465738932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,128,0,1,fp8,fp8,0,6.81385612487793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,128,0,1,float16,fp8,0,9.975690841674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,128,0,1,fp8,fp8,0,6.735232035319011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,128,0,1,float16,float16,0,10.218778610229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,128,0,1,float16,fp8,0,10.028495788574219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,128,0,1,float16,float16,0,9.867813110351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,128,0,1,fp8,fp8,0,7.011898676554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,128,0,1,float16,fp8,0,9.983370463053385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,128,0,1,fp8,fp8,0,6.972719828287761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,128,0,1,fp8,fp8,0,3.6531041463216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,128,0,1,float16,float16,0,5.393648147583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,128,0,1,float16,fp8,0,5.2889814376831055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,128,0,1,float16,float16,0,10.162389119466146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,128,0,1,float16,fp8,0,9.760794957478842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,128,0,1,float16,float16,0,5.3002932866414385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,128,0,1,fp8,fp8,0,3.4209121068318686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,128,0,1,fp8,fp8,0,3.4369920094807944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,128,0,1,float16,fp8,0,5.089205423990886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,128,0,1,float16,float16,0,5.205258687337239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,128,0,1,float16,fp8,0,5.077424049377441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,128,0,1,fp8,fp8,0,3.803189277648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,128,0,1,float16,fp8,0,5.103061358133952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,128,0,1,float16,float16,0,5.084469477335612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,128,0,1,fp8,fp8,0,3.431034723917643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,128,0,1,float16,float16,0,5.071615854899089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,128,0,1,float16,fp8,0,5.954138437906901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,128,0,1,fp8,fp8,0,19.206527709960938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,128,0,1,fp8,fp8,0,19.075531005859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,128,0,1,float16,fp8,0,28.33490244547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,128,0,1,float16,float16,0,27.991343180338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,128,0,1,float16,float16,0,27.396224975585938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,128,0,1,float16,fp8,0,28.02319590250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,128,0,1,float16,float16,0,28.180068969726562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,128,0,1,float16,fp8,0,26.821317036946613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,128,0,1,fp8,fp8,0,18.962212880452473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,128,0,1,fp8,fp8,0,9.79531733194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,128,0,1,float16,float16,0,14.586037953694662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,128,0,1,float16,fp8,0,13.902596791585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,128,0,1,fp8,fp8,0,18.828795115152996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,128,0,1,float16,float16,0,27.596405029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,128,0,1,fp8,fp8,0,9.445648193359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,128,0,1,float16,fp8,0,14.366043090820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,128,0,1,float16,float16,0,14.376079559326172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,128,0,1,float16,fp8,0,27.97545115152995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,128,0,1,fp8,fp8,0,9.682533264160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,128,0,1,float16,float16,0,13.891952514648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,128,0,1,float16,fp8,0,13.947125752766928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,128,0,1,fp8,fp8,0,9.716853459676107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,128,0,1,float16,float16,0,13.965349833170572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,128,0,1,float16,float16,0,7.192165374755859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,128,0,1,float16,fp8,0,14.01520029703776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,128,0,1,float16,fp8,0,7.316272099812825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,128,0,1,float16,float16,0,13.590021769205729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,128,0,1,fp8,fp8,0,9.817269643147787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,128,0,1,fp8,fp8,0,5.202255884806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,128,0,1,float16,fp8,0,14.142218271891275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,128,0,1,float16,float16,0,7.028730392456055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,128,0,1,fp8,fp8,0,4.8622134526570635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,128,0,1,float16,fp8,0,7.003765106201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,128,0,1,fp8,fp8,0,4.745696067810059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,128,0,1,float16,float16,0,6.842367808024089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,128,0,1,float16,fp8,0,6.862239837646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,128,0,1,fp8,fp8,0,4.785909334818522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,128,0,1,float16,float16,0,7.08787218729655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,128,0,1,fp8,fp8,0,4.763813336690267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,128,0,1,float16,fp8,0,7.084826787312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,128,0,1,float16,float16,0,3.653845469156901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,128,0,1,float16,fp8,0,3.939509391784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,128,0,1,fp8,fp8,0,2.5527572631835938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,128,0,1,float16,float16,0,7.047594706217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,128,0,1,float16,fp8,0,7.217274983723958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,128,0,1,float16,float16,0,3.9675734837849936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,128,0,1,fp8,fp8,0,2.3966026306152344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,128,0,1,float16,fp8,0,3.5158987045288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,128,0,1,fp8,fp8,0,2.4757973353068032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,128,0,1,float16,float16,0,3.4943412144978843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,128,0,1,float16,fp8,0,4.051007906595866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,128,0,1,float16,float16,0,3.6223039627075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,128,0,1,fp8,fp8,0,2.419274648030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,128,0,1,float16,fp8,0,3.6677068074544272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,128,0,1,fp8,fp8,0,2.4836479822794595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,128,0,1,float16,float16,0,3.7492586771647134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,128,0,1,float16,fp8,0,3.713541348775228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,128,0,1,fp8,fp8,0,25.03063456217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,128,0,1,fp8,fp8,0,25.144388834635418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,128,0,1,float16,float16,0,35.739105224609375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,128,0,1,float16,fp8,0,37.90476735432943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,128,0,1,float16,fp8,0,36.73799387613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,128,0,1,float16,float16,0,37.44202168782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,128,0,1,float16,float16,0,36.82533264160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,128,0,1,float16,fp8,0,36.359082539876304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,128,0,1,fp8,fp8,0,25.553695678710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,128,0,1,fp8,fp8,0,13.463136037190756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,128,0,1,float16,float16,0,18.797754923502605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,128,0,1,float16,fp8,0,19.560357411702473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,128,0,1,float16,float16,0,18.61837895711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,128,0,1,fp8,fp8,0,24.82813262939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,128,0,1,float16,float16,0,36.92411804199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,128,0,1,fp8,fp8,0,12.784219106038412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,128,0,1,float16,fp8,0,18.233440399169922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,128,0,1,fp8,fp8,0,12.59655507405599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,128,0,1,float16,fp8,0,36.270790100097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,128,0,1,float16,float16,0,18.277562459309895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,128,0,1,float16,fp8,0,18.749839782714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,128,0,1,fp8,fp8,0,12.611099243164062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,128,0,1,float16,float16,0,18.798255920410156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,128,0,1,float16,fp8,0,17.990880330403645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,128,0,1,fp8,fp8,0,12.71542994181315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,128,0,1,float16,float16,0,9.61296017964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,128,0,1,fp8,fp8,0,6.897621154785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,128,0,1,float16,fp8,0,9.779354731241861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,128,0,1,float16,float16,0,18.697551727294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,128,0,1,float16,fp8,0,18.77872594197591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,128,0,1,float16,float16,0,8.982389450073242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,128,0,1,float16,fp8,0,9.201103846232096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,128,0,1,fp8,fp8,0,6.346389134724935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,128,0,1,fp8,fp8,0,6.379536310831706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,128,0,1,float16,float16,0,9.097882588704428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,128,0,1,float16,fp8,0,9.342565536499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,128,0,1,fp8,fp8,0,6.266037623087565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,128,0,1,float16,float16,0,9.142570495605469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,128,0,1,float16,fp8,0,9.123573303222656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,128,0,1,fp8,fp8,0,6.533583958943685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,128,0,1,float16,float16,0,4.811648050944011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,128,0,1,float16,fp8,0,4.983850797017415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,128,0,1,fp8,fp8,0,3.425168037414551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,128,0,1,float16,float16,0,9.350106557210287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,128,0,1,float16,fp8,0,9.169685363769531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,128,0,1,float16,float16,0,4.650202751159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,128,0,1,fp8,fp8,0,3.2312211990356445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,128,0,1,fp8,fp8,0,3.2022988001505532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,128,0,1,float16,fp8,0,4.773082733154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,128,0,1,float16,float16,0,4.70797856648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,128,0,1,float16,float16,0,4.71669864654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,128,0,1,float16,fp8,0,5.075242678324382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,128,0,1,fp8,fp8,0,3.2276426951090493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,128,0,1,float16,fp8,0,4.692720095316569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,128,0,1,float16,float16,0,2.4305760065714517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,128,0,1,fp8,fp8,0,1.719098726908366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,128,0,1,float16,fp8,0,2.409242630004883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,128,0,1,fp8,fp8,0,3.249706586201986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,128,0,1,float16,float16,0,4.686890602111816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,128,0,1,float16,fp8,0,4.62278938293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,128,0,1,float16,float16,0,2.373967965443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,128,0,1,fp8,fp8,0,1.613104025522868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,128,0,1,float16,fp8,0,2.3173440297444663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,128,0,1,fp8,fp8,0,1.6288746198018391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,128,0,1,float16,float16,0,2.391162713368734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,128,0,1,float16,fp8,0,2.309055964152018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,128,0,1,float16,float16,0,2.352549393971761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,128,0,1,fp8,fp8,0,1.638746738433838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,128,0,1,float16,fp8,0,2.305173397064209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,128,0,1,float16,float16,0,2.4383947054545083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,128,0,1,fp8,fp8,0,1.69758939743042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,128,0,1,float16,fp8,0,2.4607200622558594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,128,0,1,fp8,fp8,0,15.271184285481771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,128,0,1,fp8,fp8,0,15.03863525390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,128,0,1,float16,float16,0,21.243834177652996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,128,0,1,float16,fp8,0,21.895060221354168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,128,0,1,float16,float16,0,21.607999165852863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,128,0,1,float16,fp8,0,20.983333587646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,128,0,1,float16,float16,0,21.691001892089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,128,0,1,float16,fp8,0,21.491012573242188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,128,0,1,fp8,fp8,0,15.177738189697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,128,0,1,fp8,fp8,0,8.077018737792969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,128,0,1,float16,float16,0,11.83084233601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,128,0,1,float16,float16,0,10.757493336995443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,128,0,1,float16,fp8,0,13.648576100667318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,128,0,1,fp8,fp8,0,15.02231470743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,128,0,1,float16,float16,0,21.77081553141276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,128,0,1,fp8,fp8,0,7.489845275878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,128,0,1,float16,fp8,0,10.741765340169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,128,0,1,fp8,fp8,0,7.622117360432942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,128,0,1,float16,fp8,0,21.735328674316406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,128,0,1,float16,float16,0,10.66270383199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,128,0,1,float16,fp8,0,10.767147064208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,128,0,1,fp8,fp8,0,7.417146682739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,128,0,1,float16,float16,0,10.743125915527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,128,0,1,float16,fp8,0,10.76840591430664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,128,0,1,float16,float16,0,5.783775965372722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,128,0,1,fp8,fp8,0,7.500245412190755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,128,0,1,float16,fp8,0,5.645743687947591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,128,0,1,fp8,fp8,0,4.11790402730306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,128,0,1,float16,float16,0,10.917621612548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,128,0,1,float16,float16,0,5.452565511067708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,128,0,1,float16,fp8,0,10.34611701965332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,128,0,1,float16,fp8,0,5.4938398996988935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,128,0,1,fp8,fp8,0,3.7261759440104165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,128,0,1,fp8,fp8,0,3.829845428466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,128,0,1,float16,fp8,0,5.480415980021159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,128,0,1,float16,float16,0,5.481615702311198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,128,0,1,fp8,fp8,0,3.6446186701456704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,128,0,1,float16,float16,0,5.438240051269531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,128,0,1,float16,fp8,0,5.5033919016520185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,128,0,1,fp8,fp8,0,3.819712003072103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,128,0,1,float16,float16,0,2.842106819152832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,128,0,1,float16,fp8,0,2.6612960497538247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,128,0,1,fp8,fp8,0,2.0244852701822915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,128,0,1,float16,float16,0,5.357434590657552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,128,0,1,float16,fp8,0,5.504624048868815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,128,0,1,float16,float16,0,2.702058792114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,128,0,1,fp8,fp8,0,1.8533066113789876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,128,0,1,float16,fp8,0,2.71562131245931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,128,0,1,fp8,fp8,0,1.863690694173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,128,0,1,float16,float16,0,2.719285329182943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,128,0,1,float16,fp8,0,2.786261240641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,128,0,1,float16,float16,0,2.756592114766439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,128,0,1,float16,fp8,0,2.819354693094889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,128,0,1,fp8,fp8,0,1.944709300994873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,128,0,1,float16,float16,0,2.520784060160319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,128,0,1,fp8,fp8,0,1.8622345924377441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,128,0,1,float16,float16,0,1.4698079427083333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,128,0,1,fp8,fp8,0,1.117184003194173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,128,0,1,float16,fp8,0,1.4846240679423015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,128,0,1,float16,fp8,0,2.794149398803711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,128,0,1,float16,float16,0,1.3822239240010579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,128,0,1,float16,fp8,0,1.3608320554097493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,128,0,1,fp8,fp8,0,1.1093440055847168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,128,0,1,fp8,fp8,0,1.0097760359446208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,128,0,1,float16,float16,0,1.4880266189575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,128,0,1,float16,fp8,0,1.3708426157633464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,128,0,1,fp8,fp8,0,1.0478986899058025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,128,0,1,float16,float16,0,1.412783940633138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,128,0,1,float16,float16,0,1.3746293385823567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,128,0,1,float16,fp8,0,1.4274293581644695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,128,0,1,fp8,fp8,0,1.0101226965586345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,128,0,1,float16,fp8,0,1.3997333844502766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,128,0,1,fp8,fp8,0,14.805408477783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,128,0,1,fp8,fp8,0,14.747066497802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,128,0,1,float16,fp8,0,21.008533477783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,128,0,1,float16,float16,0,20.82797368367513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,128,0,1,float16,float16,0,21.138277689615887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,128,0,1,float16,fp8,0,21.127957661946613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,128,0,1,float16,float16,0,20.873631795247395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,128,0,1,fp8,fp8,0,15.267312367757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,128,0,1,float16,fp8,0,20.018282572428387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,128,0,1,fp8,fp8,0,8.123141606648764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,128,0,1,float16,float16,0,11.39785639444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,128,0,1,float16,fp8,0,11.56451670328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,128,0,1,fp8,fp8,0,15.213845570882162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,128,0,1,float16,float16,0,21.58787790934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,128,0,1,fp8,fp8,0,7.401034673055013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,128,0,1,float16,fp8,0,10.557680130004883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,128,0,1,float16,float16,0,10.359029134114584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,128,0,1,float16,fp8,0,21.018938700358074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,128,0,1,float16,float16,0,10.296143849690756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,128,0,1,float16,fp8,0,10.287061055501303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,128,0,1,fp8,fp8,0,7.680474599202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,128,0,1,float16,float16,0,10.266543706258139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,128,0,1,float16,fp8,0,10.50921630859375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,128,0,1,fp8,fp8,0,7.923061370849609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,128,0,1,float16,float16,0,5.431397120157878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,128,0,1,float16,fp8,0,6.097925186157227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,128,0,1,fp8,fp8,0,7.533807754516602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,128,0,1,fp8,fp8,0,4.063797314961751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,128,0,1,float16,float16,0,10.373866399129232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,128,0,1,float16,fp8,0,10.443503697713217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,128,0,1,float16,float16,0,5.204869270324707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,128,0,1,fp8,fp8,0,3.6273654301961265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,128,0,1,float16,fp8,0,5.0695145924886065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,128,0,1,fp8,fp8,0,3.6110188166300454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,128,0,1,float16,float16,0,5.259418805440267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,128,0,1,float16,fp8,0,5.303226788838704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,128,0,1,float16,float16,0,5.082709312438965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,128,0,1,float16,fp8,0,5.073952039082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,128,0,1,fp8,fp8,0,3.6237548192342124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,128,0,1,float16,float16,0,2.7593278884887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,128,0,1,float16,float16,0,5.311312039693196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,128,0,1,fp8,fp8,0,2.027359962463379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,128,0,1,fp8,fp8,0,3.7749013900756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,128,0,1,float16,fp8,0,2.802879969278971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,128,0,1,float16,fp8,0,4.9927412668863935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,128,0,1,float16,float16,0,2.5238720575968423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,128,0,1,float16,fp8,0,2.5173120498657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,128,0,1,fp8,fp8,0,1.8783626556396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,128,0,1,fp8,fp8,0,1.821781317392985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,128,0,1,float16,float16,0,2.562117258707682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,128,0,1,fp8,fp8,0,1.8820692698160808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,128,0,1,float16,fp8,0,2.659301280975342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,128,0,1,float16,float16,0,2.5674826304117837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,128,0,1,float16,fp8,0,2.5002506573994956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,128,0,1,float16,float16,0,2.6703945795694985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,128,0,1,fp8,fp8,0,1.1054879824320476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,128,0,1,fp8,fp8,0,1.8245333035786946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,128,0,1,float16,float16,0,1.4226080576578777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,128,0,1,float16,fp8,0,1.387455940246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,128,0,1,float16,fp8,0,2.5082133611043296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,128,0,1,float16,float16,0,1.3291359742482503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,128,0,1,float16,fp8,0,1.3263200124104817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,128,0,1,fp8,fp8,0,1.0045013427734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,128,0,1,fp8,fp8,0,0.9609440167744955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,128,0,1,float16,float16,0,1.2895413239796956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,128,0,1,float16,fp8,0,1.3237919807434082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,128,0,1,fp8,fp8,0,0.9470986525217692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,128,0,1,float16,float16,0,1.2812373638153076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,128,0,1,float16,fp8,0,1.2945653597513835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,128,0,1,float16,float16,0,1.2728373209635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,128,0,1,float16,fp8,0,1.2714880307515461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,128,0,1,float16,float16,0,0.774288018544515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,128,0,1,fp8,fp8,0,0.9622560342152914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,128,0,1,float16,fp8,0,0.782591978708903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,128,0,1,fp8,fp8,0,0.5703200101852417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,128,0,1,float16,float16,0,0.6928640206654867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,128,0,1,fp8,fp8,0,0.5404586791992188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,128,0,1,float16,fp8,0,0.7028053601582845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,128,0,1,float16,float16,0,0.7368000348409017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,128,0,1,fp8,fp8,0,0.5663466850916544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,128,0,1,float16,fp8,0,0.7702613671620687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,128,0,1,float16,float16,0,0.7358559767405192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,128,0,1,float16,fp8,0,0.6946986516316732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,128,0,1,fp8,fp8,0,0.5385760068893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,128,0,1,float16,float16,0,0.6929333209991455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,128,0,1,fp8,fp8,0,0.5506026744842529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,128,0,1,float16,fp8,0,0.7283519903818766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,128,0,1,fp8,fp8,0,9.179242451985678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,128,0,1,fp8,fp8,0,8.972245534261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,128,0,1,float16,float16,0,12.49072519938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,128,0,1,float16,fp8,0,12.609792073567709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,128,0,1,float16,fp8,0,12.376667022705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,128,0,1,float16,float16,0,12.374795277913412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,128,0,1,float16,float16,0,12.515263875325521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,128,0,1,float16,fp8,0,12.503584543863932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,128,0,1,fp8,fp8,0,5.219301223754883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,128,0,1,fp8,fp8,0,9.261605580647787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,128,0,1,float16,float16,0,6.8396962483723955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,128,0,1,float16,fp8,0,6.886559804280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,128,0,1,float16,float16,0,6.322746912638347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,128,0,1,fp8,fp8,0,9.344048182169596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,128,0,1,float16,float16,0,12.799914042154947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,128,0,1,fp8,fp8,0,4.6363786061604815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,128,0,1,float16,fp8,0,12.969322204589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,128,0,1,float16,fp8,0,6.272288004557292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,128,0,1,float16,float16,0,6.168938954671224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,128,0,1,float16,fp8,0,6.398138682047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,128,0,1,fp8,fp8,0,4.538613319396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,128,0,1,float16,float16,0,6.197722752888997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,128,0,1,fp8,fp8,0,4.822250684102376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,128,0,1,float16,float16,0,3.480527877807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,128,0,1,float16,fp8,0,6.479408264160156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,128,0,1,float16,float16,0,6.608373641967773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,128,0,1,fp8,fp8,0,4.790533383687337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,128,0,1,float16,fp8,0,3.6322558720906577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,128,0,1,fp8,fp8,0,2.7403411865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,128,0,1,float16,fp8,0,6.665162404378255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,128,0,1,float16,float16,0,3.2460959752400718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,128,0,1,fp8,fp8,0,2.5018506050109863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,128,0,1,float16,fp8,0,3.3107732137044272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,128,0,1,float16,float16,0,3.3470026652018228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,128,0,1,fp8,fp8,0,2.1933120091756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,128,0,1,float16,fp8,0,3.0537598927815757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,128,0,1,float16,float16,0,3.0339574813842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,128,0,1,fp8,fp8,0,2.200549284617106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,128,0,1,float16,fp8,0,2.948063850402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,128,0,1,float16,float16,0,1.7077226638793945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,128,0,1,fp8,fp8,0,2.293226718902588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,128,0,1,fp8,fp8,0,1.256223996480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,128,0,1,float16,float16,0,3.0680319468180337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,128,0,1,float16,fp8,0,3.0633974075317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,128,0,1,float16,fp8,0,1.6676534016927083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,128,0,1,float16,float16,0,1.4546987215677898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,128,0,1,float16,fp8,0,1.484063943227132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,128,0,1,fp8,fp8,0,1.2340426445007324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,128,0,1,float16,float16,0,1.4830293655395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,128,0,1,fp8,fp8,0,1.1635253429412842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,128,0,1,float16,fp8,0,1.4722720781962078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,128,0,1,float16,float16,0,1.4779946009318035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,128,0,1,fp8,fp8,0,1.12827730178833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,128,0,1,float16,fp8,0,1.4824479420979817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,128,0,1,float16,float16,0,0.8474559783935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,128,0,1,fp8,fp8,0,1.1345386505126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,128,0,1,float16,float16,0,1.4759467442830403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,128,0,1,float16,fp8,0,1.5146826108296711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,128,0,1,float16,fp8,0,0.8438133398691813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,128,0,1,fp8,fp8,0,0.6677707036336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,128,0,1,float16,float16,0,0.7705972989400228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,128,0,1,fp8,fp8,0,0.6215253273646036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,128,0,1,float16,fp8,0,0.7754613558451334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,128,0,1,fp8,fp8,0,0.6324160099029541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,128,0,1,float16,float16,0,0.7846879959106445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,128,0,1,float16,fp8,0,0.8075146675109863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,128,0,1,float16,float16,0,0.7759946982065836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,128,0,1,fp8,fp8,0,0.6224693457285563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,128,0,1,float16,fp8,0,0.7819093068440756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,128,0,1,float16,float16,0,0.7899893124898275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,128,0,1,fp8,fp8,0,0.6287200053532919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,128,0,1,float16,fp8,0,0.7841440041859945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,128,0,1,float16,float16,0,0.4691679875055949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,128,0,1,float16,fp8,0,0.4756266673405965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,128,0,1,fp8,fp8,0,0.37618664900461835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,128,0,1,float16,float16,0,0.445087989171346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,128,0,1,float16,fp8,0,0.44918398062388104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,128,0,1,fp8,fp8,0,0.34756799538930255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,128,0,1,fp8,fp8,0,0.3485493262608846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,128,0,1,float16,float16,0,0.45258665084838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,128,0,1,float16,fp8,0,0.4428960084915161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,128,0,1,fp8,fp8,0,0.34807467460632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,128,0,1,float16,float16,0,0.45453333854675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,128,0,1,float16,fp8,0,0.4550400177637736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,128,0,1,float16,float16,0,0.4564479986826579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,128,0,1,fp8,fp8,0,0.3500320116678874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,128,0,1,float16,fp8,0,0.4553973277409871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,128,0,1,fp8,fp8,0,9.481114705403646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,128,0,1,fp8,fp8,0,9.456303914388021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,128,0,1,float16,float16,0,12.534042358398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,128,0,1,float16,fp8,0,12.409136454264322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,128,0,1,float16,float16,0,12.373920440673828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,128,0,1,float16,fp8,0,12.541946411132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,128,0,1,float16,fp8,0,12.429519653320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,128,0,1,float16,float16,0,12.572176615397135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,128,0,1,fp8,fp8,0,5.448554356892903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,128,0,1,fp8,fp8,0,9.59444808959961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,128,0,1,float16,float16,0,6.914511998494466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,128,0,1,float16,float16,0,6.224053064982097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,128,0,1,float16,fp8,0,6.936239878336589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,128,0,1,fp8,fp8,0,9.670037587483725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,128,0,1,float16,float16,0,12.628363291422525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,128,0,1,fp8,fp8,0,4.8757279713948565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,128,0,1,float16,fp8,0,12.865610758463541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,128,0,1,fp8,fp8,0,4.755765279134114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,128,0,1,float16,fp8,0,6.177477518717448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,128,0,1,float16,float16,0,6.105183919270833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,128,0,1,float16,fp8,0,6.297029495239258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,128,0,1,float16,float16,0,6.640672047932942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,128,0,1,float16,fp8,0,6.199461619059245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,128,0,1,float16,fp8,0,3.537189483642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,128,0,1,fp8,fp8,0,5.034389177958171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,128,0,1,float16,float16,0,3.8670454025268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,128,0,1,fp8,fp8,0,5.568991978963216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,128,0,1,float16,fp8,0,6.9814879099528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,128,0,1,float16,float16,0,7.38365364074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,128,0,1,fp8,fp8,0,3.172821362813314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,128,0,1,fp8,fp8,0,2.57533327738444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,128,0,1,float16,float16,0,3.6103413899739585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,128,0,1,float16,fp8,0,3.724970817565918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,128,0,1,float16,float16,0,3.6314719518025718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,128,0,1,fp8,fp8,0,2.5781760215759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,128,0,1,float16,fp8,0,3.783263842264811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,128,0,1,float16,float16,0,3.218378702799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,128,0,1,float16,fp8,0,3.2055466969807944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,128,0,1,fp8,fp8,0,2.5312426884969077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,128,0,1,fp8,fp8,0,2.3853599230448403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,128,0,1,float16,float16,0,3.239199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,128,0,1,float16,fp8,0,3.1943626403808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,128,0,1,float16,float16,0,1.7954613367716472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,128,0,1,float16,fp8,0,1.756223996480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,128,0,1,fp8,fp8,0,1.3765600522359211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,128,0,1,float16,float16,0,1.4645387331644695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,128,0,1,fp8,fp8,0,1.1850026448567708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,128,0,1,float16,float16,0,1.4602880477905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,128,0,1,float16,fp8,0,1.5008373260498047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,128,0,1,fp8,fp8,0,1.1918559869130452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,128,0,1,float16,fp8,0,1.469861348470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,128,0,1,float16,float16,0,1.458778699239095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,128,0,1,fp8,fp8,0,1.2081653277079265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,128,0,1,float16,fp8,0,1.4645066261291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,128,0,1,float16,float16,0,0.8696426550547282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,128,0,1,float16,float16,0,1.4942453702290852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,128,0,1,float16,fp8,0,1.4903945922851562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,128,0,1,fp8,fp8,0,0.6359253327051798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,128,0,1,fp8,fp8,0,1.190287987391154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,128,0,1,float16,float16,0,0.7752959728240967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,128,0,1,float16,fp8,0,0.8667733669281006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,128,0,1,fp8,fp8,0,0.7139786879221598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,128,0,1,float16,float16,0,0.7665759722391764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,128,0,1,float16,fp8,0,0.7564480304718018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,128,0,1,float16,fp8,0,0.7690613269805908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,128,0,1,fp8,fp8,0,0.6375573476155599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,128,0,1,float16,float16,0,0.7726506392161051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,128,0,1,float16,fp8,0,0.7837173144022623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,128,0,1,float16,float16,0,0.7806186676025391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,128,0,1,fp8,fp8,0,0.6399146715799967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,128,0,1,float16,fp8,0,0.7846026420593262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,128,0,1,fp8,fp8,0,0.6302826801935831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,128,0,1,float16,float16,0,0.46319464842478436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,128,0,1,float16,fp8,0,0.4628213246663411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,128,0,1,fp8,fp8,0,0.39397867520650226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,128,0,1,float16,float16,0,0.42504000663757324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,128,0,1,float16,fp8,0,0.4267093340555827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,128,0,1,fp8,fp8,0,0.3551199833552043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,128,0,1,float16,fp8,0,0.42482133706410724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,128,0,1,float16,float16,0,0.42476268609364826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,128,0,1,float16,fp8,0,0.4272480010986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,128,0,1,fp8,fp8,0,0.3548159996668498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,128,0,1,float16,float16,0,0.42426665623982746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,128,0,1,fp8,fp8,0,0.35708800951639813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,128,0,1,float16,float16,0,0.4288746515909831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,128,0,1,fp8,fp8,0,0.35523200035095215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,128,0,1,float16,fp8,0,0.4282613197962443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,128,0,1,float16,float16,0,0.2650453249613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,128,0,1,float16,fp8,0,0.2697226603825887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,128,0,1,fp8,fp8,0,0.22453866402308145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,128,0,1,float16,float16,0,0.25285865863164264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,128,0,1,float16,fp8,0,0.2550826668739319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,128,0,1,fp8,fp8,0,0.20168532927831015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,128,0,1,fp8,fp8,0,0.2022613286972046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,128,0,1,float16,float16,0,0.2515679995218913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,128,0,1,float16,fp8,0,0.2531626621882121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,128,0,1,fp8,fp8,0,0.19796266158421835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,128,0,1,float16,float16,0,0.2549546758333842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,128,0,1,float16,fp8,0,0.25640533367792767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,128,0,1,float16,float16,0,0.25704000393549603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,128,0,1,float16,fp8,0,0.25699732700983685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,128,0,1,fp8,fp8,0,0.2037280003229777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,128,0,1,fp8,fp8,0,6.303066889444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,128,0,1,float16,float16,0,8.031754811604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,128,0,1,float16,fp8,0,7.8381703694661455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,128,0,1,fp8,fp8,0,6.262383778889974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,128,0,1,float16,float16,0,7.834800084431966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,128,0,1,float16,fp8,0,7.852048238118489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,128,0,1,float16,float16,0,7.880896250406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,128,0,1,float16,fp8,0,7.936442693074544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,128,0,1,fp8,fp8,0,6.313482920328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,128,0,1,fp8,fp8,0,3.636768023173014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,128,0,1,float16,float16,0,3.9273598988850913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,128,0,1,float16,float16,0,4.51365343729655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,128,0,1,float16,fp8,0,4.460122744242351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,128,0,1,fp8,fp8,0,6.435018539428711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,128,0,1,float16,float16,0,8.03769620259603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,128,0,1,float16,fp8,0,8.023114522298178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,128,0,1,fp8,fp8,0,3.0674187342325845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,128,0,1,float16,fp8,0,3.8592586517333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,128,0,1,fp8,fp8,0,3.0847838719685874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,128,0,1,float16,float16,0,3.8346986770629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,128,0,1,float16,fp8,0,3.8893868128458657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,128,0,1,float16,float16,0,3.9770078659057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,128,0,1,fp8,fp8,0,3.091856002807617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,128,0,1,float16,fp8,0,3.8417441050211587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,128,0,1,float16,float16,0,2.2045067151387534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,128,0,1,fp8,fp8,0,3.216202735900879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,128,0,1,float16,fp8,0,2.2406133015950522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,128,0,1,float16,float16,0,3.95963191986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,128,0,1,fp8,fp8,0,1.844703992207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,128,0,1,float16,fp8,0,3.9086666107177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,128,0,1,float16,float16,0,1.914197285970052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,128,0,1,fp8,fp8,0,1.5375626881917317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,128,0,1,float16,fp8,0,1.893280029296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,128,0,1,float16,float16,0,1.8771467208862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,128,0,1,fp8,fp8,0,1.5361226399739583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,128,0,1,float16,fp8,0,1.825152079264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,128,0,1,float16,float16,0,1.8426507314046223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,128,0,1,float16,fp8,0,1.8842506408691406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,128,0,1,fp8,fp8,0,1.5488959948221843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,128,0,1,float16,float16,0,1.1236693064371746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,128,0,1,float16,float16,0,1.9084533055623372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,128,0,1,fp8,fp8,0,1.5620959599812825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,128,0,1,float16,fp8,0,1.11080535252889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,128,0,1,float16,fp8,0,1.9201866785685222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,128,0,1,fp8,fp8,0,0.9558026790618896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,128,0,1,float16,float16,0,0.9255466461181641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,128,0,1,float16,fp8,0,0.9598186810811361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,128,0,1,fp8,fp8,0,0.8211893240610758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,128,0,1,float16,float16,0,0.9262293179829916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,128,0,1,float16,fp8,0,0.9337866306304932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,128,0,1,fp8,fp8,0,0.831775983174642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,128,0,1,float16,float16,0,0.9467466672261556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,128,0,1,float16,fp8,0,0.95469864209493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,128,0,1,fp8,fp8,0,0.818447987238566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,128,0,1,float16,float16,0,0.9481706619262695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,128,0,1,float16,float16,0,0.5849706729253134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,128,0,1,float16,fp8,0,0.948533296585083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,128,0,1,fp8,fp8,0,0.8096799850463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,128,0,1,float16,fp8,0,0.579423983891805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,128,0,1,fp8,fp8,0,0.5019146601359049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,128,0,1,float16,float16,0,0.499290664990743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,128,0,1,float16,fp8,0,0.5031573375066122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,128,0,1,fp8,fp8,0,0.43401598930358887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,128,0,1,float16,float16,0,0.5020106633504232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,128,0,1,float16,fp8,0,0.5216000080108643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,128,0,1,fp8,fp8,0,0.43723734219868976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,128,0,1,float16,float16,0,0.503386656443278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,128,0,1,float16,fp8,0,0.5090133349100748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,128,0,1,fp8,fp8,0,0.4398719867070516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,128,0,1,float16,float16,0,0.5013706684112549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,128,0,1,float16,fp8,0,0.5110506614049276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,128,0,1,fp8,fp8,0,0.4429813226064046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,128,0,1,float16,float16,0,0.2832746704419454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,128,0,1,float16,float16,0,0.31785066922505695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,128,0,1,float16,fp8,0,0.31383466720581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,128,0,1,fp8,fp8,0,0.2834666570027669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,128,0,1,float16,float16,0,0.2823466658592224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,128,0,1,float16,fp8,0,0.2847946683565776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,128,0,1,fp8,fp8,0,0.23081066211064658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,128,0,1,float16,fp8,0,0.2863626678784688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,128,0,1,fp8,fp8,0,0.23189866542816162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,128,0,1,float16,float16,0,0.2842400074005127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,128,0,1,fp8,fp8,0,0.23376532395680746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,128,0,1,float16,fp8,0,0.2861173351605733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,128,0,1,float16,float16,0,0.2855093280474345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,128,0,1,float16,fp8,0,0.2890026569366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,128,0,1,fp8,fp8,0,0.2335466742515564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,128,0,1,float16,float16,0,0.1802240014076233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,128,0,1,float16,fp8,0,0.18295466899871826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,128,0,1,fp8,fp8,0,0.1569760044415792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,128,0,1,fp8,fp8,0,0.1416213313738505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,128,0,1,float16,float16,0,0.16197333733240762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,128,0,1,float16,float16,0,0.16155200203259787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,128,0,1,float16,fp8,0,0.16240533192952475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,128,0,1,fp8,fp8,0,0.14256532986958823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,128,0,1,float16,float16,0,0.16179200013478598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,128,0,1,float16,fp8,0,0.16246400276819864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,128,0,1,float16,fp8,0,0.1628213326136271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,128,0,1,fp8,fp8,0,0.1430293321609497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,128,0,1,float16,float16,0,0.1614720026652018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,128,0,1,float16,fp8,0,0.16396799683570862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,128,0,1,fp8,fp8,0,0.14333333571751913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,128,0,1,fp8,fp8,0,6.773759841918945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,128,0,1,fp8,fp8,0,6.781381607055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,128,0,1,float16,fp8,0,7.979743957519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,128,0,1,float16,float16,0,7.943621317545573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,128,0,1,float16,float16,0,7.846303939819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,128,0,1,float16,float16,0,7.998661041259766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,128,0,1,float16,fp8,0,7.877397537231445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,128,0,1,float16,fp8,0,8.057631810506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,128,0,1,fp8,fp8,0,4.269573211669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,128,0,1,float16,float16,0,3.938373247782389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,128,0,1,float16,float16,0,5.074341456095378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,128,0,1,float16,fp8,0,4.977941195170085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,128,0,1,fp8,fp8,0,6.804149627685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,128,0,1,fp8,fp8,0,6.979056040445964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,128,0,1,float16,float16,0,8.141050974527994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,128,0,1,float16,fp8,0,8.13267707824707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,128,0,1,fp8,fp8,0,3.4277334213256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,128,0,1,float16,fp8,0,3.9920053482055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,128,0,1,float16,float16,0,3.984757423400879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,128,0,1,fp8,fp8,0,3.4226452509562173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,128,0,1,float16,fp8,0,3.8164428075154624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,128,0,1,float16,float16,0,3.9639145533243814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,128,0,1,fp8,fp8,0,3.4039786656697593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,128,0,1,float16,fp8,0,3.935466766357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,128,0,1,float16,float16,0,4.017530759175618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,128,0,1,float16,float16,0,2.5449973742167153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,128,0,1,fp8,fp8,0,3.5334294637044272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,128,0,1,float16,fp8,0,4.106597264607747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,128,0,1,float16,float16,0,1.9250399271647136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,128,0,1,fp8,fp8,0,2.1320746739705405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,128,0,1,float16,fp8,0,2.4693546295166016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,128,0,1,fp8,fp8,0,1.7146453857421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,128,0,1,float16,fp8,0,1.9173173904418945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,128,0,1,float16,float16,0,1.9282239278157551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,128,0,1,float16,fp8,0,1.931829293568929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,128,0,1,fp8,fp8,0,1.6797173817952473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,128,0,1,float16,fp8,0,1.9503572781880696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,128,0,1,float16,float16,0,1.9480853080749512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,128,0,1,fp8,fp8,0,1.7344053586324055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,128,0,1,float16,float16,0,1.9683839480082195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,128,0,1,float16,float16,0,1.2667360305786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,128,0,1,float16,fp8,0,1.2339200178782146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,128,0,1,fp8,fp8,0,1.0757173697153728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,128,0,1,float16,fp8,0,1.9429012934366863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,128,0,1,fp8,fp8,0,1.730112075805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,128,0,1,float16,float16,0,0.9768799940745035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,128,0,1,float16,fp8,0,0.9900053342183431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,128,0,1,fp8,fp8,0,0.8401760260264078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,128,0,1,float16,float16,0,0.966538667678833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,128,0,1,float16,fp8,0,0.9798133373260498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,128,0,1,fp8,fp8,0,0.832634687423706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,128,0,1,float16,float16,0,0.9834293524424235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,128,0,1,float16,fp8,0,0.9797546863555908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,128,0,1,fp8,fp8,0,0.8828639984130859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,128,0,1,float16,float16,0,0.644373337427775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,128,0,1,float16,float16,0,0.9965066909790039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,128,0,1,fp8,fp8,0,0.8633813063303629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,128,0,1,float16,fp8,0,0.6215999921162924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,128,0,1,float16,fp8,0,0.9924586613972982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,128,0,1,fp8,fp8,0,0.5437706708908081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,128,0,1,float16,float16,0,0.4984480142593384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,128,0,1,float16,fp8,0,0.4995306730270386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,128,0,1,fp8,fp8,0,0.4331039985020955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,128,0,1,float16,float16,0,0.5022293329238892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,128,0,1,float16,fp8,0,0.4920479853947957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,128,0,1,fp8,fp8,0,0.4337759812672933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,128,0,1,float16,float16,0,0.5014880100886027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,128,0,1,float16,fp8,0,0.5062293211619059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,128,0,1,fp8,fp8,0,0.4369973341623942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,128,0,1,float16,float16,0,0.5136586825052897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,128,0,1,float16,fp8,0,0.5107786655426025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,128,0,1,fp8,fp8,0,0.44682665665944415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,128,0,1,float16,float16,0,0.3376053174336751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,128,0,1,fp8,fp8,0,0.27949867645899457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,128,0,1,float16,fp8,0,0.26103466749191284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,128,0,1,float16,fp8,0,0.32712000608444214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,128,0,1,float16,float16,0,0.259935994942983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,128,0,1,float16,fp8,0,0.2594933311144511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,128,0,1,fp8,fp8,0,0.23053866624832153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,128,0,1,float16,float16,0,0.26146133740743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,128,0,1,fp8,fp8,0,0.2294399936993917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,128,0,1,float16,float16,0,0.2703839937845866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,128,0,1,float16,fp8,0,0.2643199960390727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,128,0,1,float16,float16,0,0.26470933357874554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,128,0,1,fp8,fp8,0,0.22845866282780966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,128,0,1,float16,fp8,0,0.27028799057006836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,128,0,1,fp8,fp8,0,0.23040533065795898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,128,0,1,float16,float16,0,0.18150933583577475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,128,0,1,float16,fp8,0,0.17697600523630777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,128,0,1,fp8,fp8,0,0.14497066537539163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,128,0,1,float16,float16,0,0.1418773333231608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,128,0,1,float16,fp8,0,0.1423466702302297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,128,0,1,fp8,fp8,0,0.11316266655921936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,128,0,1,float16,float16,0,0.1423413356145223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,128,0,1,float16,fp8,0,0.1409386694431305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,128,0,1,fp8,fp8,0,0.11396800478299458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,128,0,1,float16,float16,0,0.14733866850535074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,128,0,1,float16,fp8,0,0.14482667048772177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,128,0,1,fp8,fp8,0,0.11306132872899373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,128,0,1,float16,float16,0,0.14629866679509482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,128,0,1,float16,fp8,0,0.14659200112024942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,128,0,1,fp8,fp8,0,0.11523733536402385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,128,0,1,float16,float16,0,0.09693333506584167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,128,0,1,float16,fp8,0,0.0953653355439504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,128,0,1,fp8,fp8,0,0.08041599889596303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,128,0,1,float16,float16,0,0.0751200020313263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,128,0,1,float16,fp8,0,0.07566933333873749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,128,0,1,fp8,fp8,0,0.06191466748714447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,128,0,1,float16,float16,0,0.07427733143170674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,128,0,1,float16,fp8,0,0.07525333265463512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,128,0,1,fp8,fp8,0,0.06258133550484975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,128,0,1,float16,float16,0,0.07491733133792877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,128,0,1,float16,fp8,0,0.07585600018501282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,128,0,1,fp8,fp8,0,0.06379200021425883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,128,0,1,float16,float16,0,0.07743999858697255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,128,0,1,float16,fp8,0,0.07632533212502797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,128,0,1,fp8,fp8,0,0.0650133341550827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,128,0,1,fp8,fp8,0,5.475264231363933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,128,0,1,float16,float16,0,6.054298400878906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,128,0,1,float16,fp8,0,6.000960032145183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,128,0,1,fp8,fp8,0,5.576357523600261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,128,0,1,float16,float16,0,6.009450912475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,128,0,1,float16,fp8,0,5.91481081644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,128,0,1,float16,float16,0,6.038506825764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,128,0,1,float16,fp8,0,6.096090952555339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,128,0,1,fp8,fp8,0,3.617098808288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,128,0,1,float16,float16,0,4.270373344421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,128,0,1,float16,fp8,0,4.1487681070963545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,128,0,1,fp8,fp8,0,5.538592020670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,128,0,1,float16,float16,0,6.158976236979167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,128,0,1,fp8,fp8,0,5.708656311035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,128,0,1,float16,fp8,0,6.314751942952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,128,0,1,float16,float16,0,2.9663521448771157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,128,0,1,fp8,fp8,0,2.780837376912435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,128,0,1,float16,fp8,0,2.9731998443603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,128,0,1,float16,float16,0,2.981194814046224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,128,0,1,fp8,fp8,0,2.7340854008992515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,128,0,1,float16,fp8,0,3.0017547607421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,128,0,1,float16,float16,0,3.0101280212402344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,128,0,1,fp8,fp8,0,2.7480319341023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,128,0,1,float16,float16,0,3.055365244547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,128,0,1,float16,fp8,0,2.975653330485026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,128,0,1,float16,fp8,0,3.046191851298014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,128,0,1,float16,float16,0,2.176725387573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,128,0,1,float16,fp8,0,2.0582453409830728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,128,0,1,fp8,fp8,0,2.8823572794596353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,128,0,1,fp8,fp8,0,1.8070613543192546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,128,0,1,float16,float16,0,1.4930987358093262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,128,0,1,fp8,fp8,0,1.3683466911315918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,128,0,1,float16,fp8,0,1.5200533866882324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,128,0,1,float16,float16,0,1.4987573623657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,128,0,1,fp8,fp8,0,1.3898240725199382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,128,0,1,float16,fp8,0,1.4986826578776042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,128,0,1,float16,float16,0,1.4939200083414714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,128,0,1,float16,fp8,0,1.5043147404988606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,128,0,1,fp8,fp8,0,1.3974186579386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,128,0,1,float16,float16,0,1.5426185925801594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,128,0,1,float16,fp8,0,1.0242400169372559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,128,0,1,float16,float16,0,1.0693973700205486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,128,0,1,fp8,fp8,0,0.9083413283030192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,128,0,1,float16,fp8,0,1.5374773343404133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,128,0,1,float16,float16,0,0.7511413097381592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,128,0,1,fp8,fp8,0,1.4085119565327961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,128,0,1,float16,fp8,0,0.7600586414337158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,128,0,1,fp8,fp8,0,0.6763573487599691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,128,0,1,float16,fp8,0,0.7636533578236898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,128,0,1,float16,float16,0,0.7589173316955566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,128,0,1,fp8,fp8,0,0.6849813461303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,128,0,1,float16,float16,0,0.7709333101908366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,128,0,1,float16,fp8,0,0.7565493583679199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,128,0,1,fp8,fp8,0,0.6850240230560303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,128,0,1,float16,float16,0,0.7802240053812662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,128,0,1,float16,fp8,0,0.778170665105184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,128,0,1,fp8,fp8,0,0.7253653208414713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,128,0,1,float16,float16,0,0.3815466562906901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,128,0,1,float16,float16,0,0.537717342376709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,128,0,1,float16,fp8,0,0.5212159951527914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,128,0,1,fp8,fp8,0,0.45723732312520343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,128,0,1,float16,fp8,0,0.3821866512298584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,128,0,1,fp8,fp8,0,0.3487146695454915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,128,0,1,float16,float16,0,0.39050666491190594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,128,0,1,float16,fp8,0,0.3906879822413127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,128,0,1,fp8,fp8,0,0.35283199946085614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,128,0,1,fp8,fp8,0,0.3459999958674113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,128,0,1,float16,float16,0,0.3914346694946289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,128,0,1,float16,fp8,0,0.39339200655619305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,128,0,1,float16,float16,0,0.3960213263829549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,128,0,1,float16,fp8,0,0.40165332953135174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,128,0,1,float16,float16,0,0.20004266500473022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,128,0,1,fp8,fp8,0,0.35947199662526447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,128,0,1,float16,float16,0,0.2824853261311849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,128,0,1,float16,fp8,0,0.2741066614786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,128,0,1,fp8,fp8,0,0.2368639906247457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,128,0,1,float16,fp8,0,0.20272533098856607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,128,0,1,fp8,fp8,0,0.18121065696080527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,128,0,1,float16,fp8,0,0.2114186684290568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,128,0,1,float16,float16,0,0.20405866702397665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,128,0,1,float16,fp8,0,0.20270399252573648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,128,0,1,fp8,fp8,0,0.1856693426767985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,128,0,1,float16,float16,0,0.20729066928227743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,128,0,1,fp8,fp8,0,0.18780799706776938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,128,0,1,float16,float16,0,0.2123946746190389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,128,0,1,float16,fp8,0,0.20808533827463785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,128,0,1,fp8,fp8,0,0.18915200233459473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,128,0,1,float16,float16,0,0.1516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,128,0,1,float16,fp8,0,0.1467359960079193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,128,0,1,fp8,fp8,0,0.1283253331979116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,128,0,1,float16,float16,0,0.11029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,128,0,1,float16,fp8,0,0.11216533184051514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,128,0,1,fp8,fp8,0,0.09462933739026387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,128,0,1,float16,float16,0,0.11219200491905212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,128,0,1,float16,fp8,0,0.11157866319020589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,128,0,1,fp8,fp8,0,0.0953653355439504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,128,0,1,float16,float16,0,0.11201600233713786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,128,0,1,float16,fp8,0,0.11219732960065205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,128,0,1,float16,fp8,0,0.0842080016930898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,128,0,1,fp8,fp8,0,0.09515200058619182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,128,0,1,float16,float16,0,0.11711466312408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,128,0,1,float16,fp8,0,0.11326932907104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,128,0,1,fp8,fp8,0,0.09770666559537251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,128,0,1,float16,float16,0,0.08630933364232381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,128,0,1,fp8,fp8,0,0.07141333321730296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,128,0,1,float16,float16,0,0.060271998246510826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,128,0,1,float16,fp8,0,0.061978667974472046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,128,0,1,fp8,fp8,0,0.052757332722345986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,128,0,1,float16,fp8,0,0.06133866806825002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,128,0,1,float16,float16,0,0.06098666787147522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,128,0,1,float16,fp8,0,0.06062933305899302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,128,0,1,fp8,fp8,0,0.05336533486843109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,128,0,1,float16,float16,0,0.061797335743904114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,128,0,1,fp8,fp8,0,0.05468800167242686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,128,0,1,float16,float16,0,0.06314133107662201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,128,0,1,float16,fp8,0,0.062080000837643944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,128,0,1,fp8,fp8,0,0.05526400109132131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,128,0,1,float16,float16,0,0.04368533194065094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,128,0,1,float16,fp8,0,0.04275199770927429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,128,0,1,float16,fp8,0,0.036415999134381614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,128,0,1,fp8,fp8,0,0.04135466615358988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,128,0,1,float16,float16,0,0.036288000643253326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,128,0,1,float16,fp8,0,0.036917333801587425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,128,0,1,fp8,fp8,0,0.03086400032043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,128,0,1,float16,float16,0,0.03603200117746989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,128,0,1,fp8,fp8,0,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,128,0,1,float16,float16,0,0.03663466622432073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,128,0,1,float16,fp8,0,0.03754133234421412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,128,0,1,fp8,fp8,0,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,128,0,1,float16,float16,0,0.0367253323396047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,128,0,1,float16,fp8,0,0.037258667250474296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,128,0,1,fp8,fp8,0,0.03357866654793421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,128,0,1,fp8,fp8,0,2.1984052658081055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,128,0,1,float16,float16,0,2.532591978708903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,128,0,1,float16,fp8,0,2.5301440556844077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,128,0,1,fp8,fp8,0,2.1843412717183432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,128,0,1,float16,float16,0,2.5376052856445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,128,0,1,float16,fp8,0,2.5550293922424316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,128,0,1,float16,float16,0,2.5587573051452637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,128,0,1,fp8,fp8,0,2.263930638631185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,128,0,1,float16,float16,0,1.9588799476623535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,128,0,1,float16,fp8,0,2.5485973358154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,128,0,1,fp8,fp8,0,2.312485376993815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,128,0,1,float16,fp8,0,1.884501298268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,128,0,1,float16,float16,0,2.6182613372802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,128,0,1,float16,fp8,0,2.603125254313151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,128,0,1,float16,float16,0,1.2845173676808674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,128,0,1,fp8,fp8,0,1.0953973134358723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,128,0,1,float16,fp8,0,1.2747946580251057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,128,0,1,float16,float16,0,1.2840426762898762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,128,0,1,fp8,fp8,0,1.5697654088338215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,128,0,1,fp8,fp8,0,1.0855039755503337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,128,0,1,float16,fp8,0,1.2803253332773845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,128,0,1,float16,float16,0,1.2917813460032146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,128,0,1,fp8,fp8,0,1.1255306402842205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,128,0,1,float16,fp8,0,1.2941866715749104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,128,0,1,fp8,fp8,0,1.1480533281962078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,128,0,1,float16,float16,0,1.3156479994455974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,128,0,1,float16,fp8,0,1.3204692999521892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,128,0,1,float16,float16,0,0.9874133268992106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,128,0,1,float16,fp8,0,0.9374826749165853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,128,0,1,float16,float16,0,0.6531253258387247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,128,0,1,float16,fp8,0,0.6542026599248251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,128,0,1,fp8,fp8,0,0.7889280319213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,128,0,1,fp8,fp8,0,0.5353173414866129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,128,0,1,float16,fp8,0,0.6556533177693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,128,0,1,float16,float16,0,0.6498560110727946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,128,0,1,fp8,fp8,0,0.5329333146413168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,128,0,1,float16,float16,0,0.6560906569163004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,128,0,1,float16,fp8,0,0.6590240001678467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,128,0,1,fp8,fp8,0,0.5500906705856323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,128,0,1,float16,float16,0,0.6726293563842773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,128,0,1,fp8,fp8,0,0.39321601390838623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,128,0,1,float16,float16,0,0.496613343556722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,128,0,1,float16,fp8,0,0.4774293502171834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,128,0,1,fp8,fp8,0,0.5855786800384521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,128,0,1,float16,fp8,0,0.6741973559061686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,128,0,1,float16,float16,0,0.33154133955637616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,128,0,1,float16,fp8,0,0.33104532957077026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,128,0,1,float16,fp8,0,0.33036800225575763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,128,0,1,fp8,fp8,0,0.27453333139419556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,128,0,1,float16,float16,0,0.3319466710090637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,128,0,1,fp8,fp8,0,0.2752586603164673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,128,0,1,float16,float16,0,0.3385973374048869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,128,0,1,float16,fp8,0,0.34033600489298504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,128,0,1,fp8,fp8,0,0.2837280035018921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,128,0,1,float16,float16,0,0.34355731805165607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,128,0,1,float16,fp8,0,0.34300800164540607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,128,0,1,fp8,fp8,0,0.2878986597061157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,128,0,1,float16,fp8,0,0.17301867405573526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,128,0,1,float16,float16,0,0.25987199942270917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,128,0,1,float16,fp8,0,0.25135467449824017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,128,0,1,fp8,fp8,0,0.20784000555674234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,128,0,1,float16,float16,0,0.17250667015711466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,128,0,1,fp8,fp8,0,0.14663466811180115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,128,0,1,float16,float16,0,0.171834667523702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,128,0,1,float16,fp8,0,0.1738026738166809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,128,0,1,fp8,fp8,0,0.1500746707121531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,128,0,1,float16,float16,0,0.17585599422454834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,128,0,1,float16,fp8,0,0.1765600045522054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,128,0,1,fp8,fp8,0,0.1516693333784739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,128,0,1,float16,fp8,0,0.13693867127100626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,128,0,1,float16,float16,0,0.18303465843200684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,128,0,1,float16,fp8,0,0.18018666903177896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,128,0,1,fp8,fp8,0,0.15282666683197021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,128,0,1,float16,float16,0,0.1405280033747355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,128,0,1,fp8,fp8,0,0.11942399541536967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,128,0,1,fp8,fp8,0,0.08369599779446919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,128,0,1,float16,float16,0,0.09409067034721375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,128,0,1,float16,fp8,0,0.09576533238093059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,128,0,1,fp8,fp8,0,0.08381332953770955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,128,0,1,float16,float16,0,0.09558932979901631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,128,0,1,float16,fp8,0,0.09537067015965779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,128,0,1,float16,fp8,0,0.0981173316637675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,128,0,1,float16,float16,0,0.09744532903035481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,128,0,1,float16,fp8,0,0.09673066933949788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,128,0,1,fp8,fp8,0,0.0841493308544159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,128,0,1,float16,float16,0,0.09903466701507568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,128,0,1,fp8,fp8,0,0.08774933218955994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,128,0,1,float16,float16,0,0.08453866839408875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,128,0,1,float16,fp8,0,0.08214400211970012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,128,0,1,fp8,fp8,0,0.06764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,128,0,1,float16,float16,0,0.055642664432525635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,128,0,1,float16,fp8,0,0.05595199763774872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,128,0,1,fp8,fp8,0,0.04739200075467428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,128,0,1,float16,float16,0,0.055973331133524575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,128,0,1,float16,fp8,0,0.05628266433874766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,128,0,1,fp8,fp8,0,0.047610665361086525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,128,0,1,float16,float16,0,0.056186666091283165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,128,0,1,float16,fp8,0,0.05717333157857259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,128,0,1,fp8,fp8,0,0.04821333289146423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,128,0,1,float16,float16,0,0.05663999915122986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,128,0,1,float16,fp8,0,0.057717333237330117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,128,0,1,fp8,fp8,0,0.05013866722583771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,128,0,1,float16,float16,0,0.04001600046952566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,128,0,1,float16,fp8,0,0.03824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,128,0,1,fp8,fp8,0,0.03809600075085958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,128,0,1,float16,float16,0,0.03259200106064478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,128,0,1,float16,fp8,0,0.03327466547489166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,128,0,1,fp8,fp8,0,0.02882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,128,0,1,float16,float16,0,0.03349866718053818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,128,0,1,float16,fp8,0,0.03310933212439219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,128,0,1,float16,fp8,0,0.03455466777086258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,128,0,1,fp8,fp8,0,0.02918400118748347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,128,0,1,fp8,fp8,0,0.0310506671667099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,128,0,1,float16,float16,0,0.03366933266321818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,128,0,1,float16,fp8,0,0.02497066557407379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,128,0,1,fp8,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,128,0,1,float16,fp8,0,0.03363733241955439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,128,0,1,fp8,fp8,0,0.0308693324526151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,128,0,1,float16,float16,0,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,128,0,1,float16,float16,0,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,128,0,1,float16,float16,0,0.023413332800070446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,128,0,1,float16,fp8,0,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,128,0,1,fp8,fp8,0,0.020618667205174763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,128,0,1,float16,float16,0,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,128,0,1,float16,fp8,0,0.023711999257405598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,128,0,1,fp8,fp8,0,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,128,0,1,float16,float16,0,0.02330133318901062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,128,0,1,float16,fp8,0,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,128,0,1,fp8,fp8,0,0.021589333812395733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,128,0,1,float16,float16,0,0.023845332364241283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,128,0,1,float16,fp8,0,0.024362665911515553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,128,0,1,fp8,fp8,0,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,128,0,1,float16,float16,0,1.203658660252889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,128,0,1,float16,fp8,0,1.2028106848398845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,128,0,1,fp8,fp8,0,1.0765066941579182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,128,0,1,float16,float16,0,1.2071733474731445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,128,0,1,fp8,fp8,0,1.0873760382334392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,128,0,1,float16,fp8,0,1.2047253449757893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,128,0,1,float16,float16,0,1.2111893494923909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,128,0,1,float16,fp8,0,1.2096426486968994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,128,0,1,fp8,fp8,0,1.1197280089060466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,128,0,1,float16,float16,0,1.244815985361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,128,0,1,float16,fp8,0,1.2389813264211018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,128,0,1,float16,float16,0,1.0032587051391602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,128,0,1,fp8,fp8,0,0.7809973557790121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,128,0,1,float16,fp8,0,0.9568373362223307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,128,0,1,fp8,fp8,0,1.1551946798960369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,128,0,1,float16,float16,0,0.612549344698588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,128,0,1,float16,fp8,0,0.6135253508885702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,128,0,1,fp8,fp8,0,0.5257440010706583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,128,0,1,float16,float16,0,0.6167146762212118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,128,0,1,float16,fp8,0,0.6155893405278524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,128,0,1,fp8,fp8,0,0.5328906774520874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,128,0,1,fp8,fp8,0,0.5526453256607056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,128,0,1,float16,float16,0,0.6181919972101847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,128,0,1,float16,fp8,0,0.6144479910532633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,128,0,1,float16,float16,0,0.6329973141352335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,128,0,1,float16,fp8,0,0.6309866507848104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,128,0,1,fp8,fp8,0,0.5790506601333618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,128,0,1,float16,float16,0,0.5110613505045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,128,0,1,float16,fp8,0,0.4912480115890503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,128,0,1,fp8,fp8,0,0.3891679843266805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,128,0,1,float16,float16,0,0.31297600269317627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,128,0,1,float16,fp8,0,0.3102560043334961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,128,0,1,fp8,fp8,0,0.2728000084559123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,128,0,1,float16,float16,0,0.3123786648114522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,128,0,1,float16,float16,0,0.3197759985923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,128,0,1,float16,fp8,0,0.3153333266576131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,128,0,1,fp8,fp8,0,0.27400533358256024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,128,0,1,float16,fp8,0,0.31666133801142377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,128,0,1,fp8,fp8,0,0.2797866662343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,128,0,1,float16,float16,0,0.3261920015017192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,128,0,1,float16,fp8,0,0.32602133353551227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,128,0,1,float16,float16,0,0.26577599843343097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,128,0,1,fp8,fp8,0,0.28656532367070514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,128,0,1,float16,fp8,0,0.261680006980896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,128,0,1,fp8,fp8,0,0.20501333475112915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,128,0,1,float16,float16,0,0.1593226691087087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,128,0,1,float16,fp8,0,0.15891200304031372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,128,0,1,fp8,fp8,0,0.14598400394121805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,128,0,1,float16,float16,0,0.16317866245905557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,128,0,1,float16,fp8,0,0.16156799594561258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,128,0,1,fp8,fp8,0,0.14749333262443542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,128,0,1,float16,float16,0,0.16621866822242737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,128,0,1,float16,fp8,0,0.1673813263575236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,128,0,1,fp8,fp8,0,0.14800000190734863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,128,0,1,float16,float16,0,0.17313599586486816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,128,0,1,float16,fp8,0,0.17298134167989096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,128,0,1,fp8,fp8,0,0.15180266896883646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,128,0,1,float16,float16,0,0.14441600441932678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,128,0,1,float16,fp8,0,0.13834666212399802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,128,0,1,fp8,fp8,0,0.11347200473149617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,128,0,1,float16,float16,0,0.08973866701126099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,128,0,1,float16,fp8,0,0.09102933605511983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,128,0,1,fp8,fp8,0,0.08363200227419536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,128,0,1,float16,float16,0,0.090938667456309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,128,0,1,float16,fp8,0,0.09172800183296204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,128,0,1,fp8,fp8,0,0.08406933148701985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,128,0,1,float16,fp8,0,0.0940053363641103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,128,0,1,float16,float16,0,0.0921013355255127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,128,0,1,float16,fp8,0,0.09194133679072063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,128,0,1,fp8,fp8,0,0.08360532919565837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,128,0,1,float16,float16,0,0.09458667039871216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,128,0,1,fp8,fp8,0,0.08654933174451192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,128,0,1,float16,float16,0,0.08220799763997395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,128,0,1,float16,fp8,0,0.07916800181070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,128,0,1,fp8,fp8,0,0.06642666459083557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,128,0,1,float16,float16,0,0.052005335688591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,128,0,1,float16,fp8,0,0.0528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,128,0,1,fp8,fp8,0,0.0473280002673467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,128,0,1,float16,fp8,0,0.05349333087603251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,128,0,1,float16,float16,0,0.052906667192777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,128,0,1,float16,fp8,0,0.052416001756985985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,128,0,1,float16,fp8,0,0.05446400245030721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,128,0,1,fp8,fp8,0,0.046810666720072426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,128,0,1,float16,float16,0,0.053727999329566956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,128,0,1,fp8,fp8,0,0.04754666487375895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,128,0,1,float16,float16,0,0.054885332783063255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,128,0,1,fp8,fp8,0,0.04910933474699656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,128,0,1,float16,float16,0,0.03825066735347112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,128,0,1,float16,fp8,0,0.03803733239571253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,128,0,1,fp8,fp8,0,0.028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,128,0,1,fp8,fp8,0,0.03755733370780945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,128,0,1,float16,float16,0,0.03169066707293192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,128,0,1,float16,fp8,0,0.03188266605138779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,128,0,1,fp8,fp8,0,0.028560000161329906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,128,0,1,float16,float16,0,0.03275199979543686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,128,0,1,float16,float16,0,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,128,0,1,float16,fp8,0,0.03190933416287104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,128,0,1,float16,float16,0,0.032586666444937386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,128,0,1,float16,fp8,0,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,128,0,1,fp8,fp8,0,0.0303413321574529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,128,0,1,float16,fp8,0,0.032826667030652366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,128,0,1,fp8,fp8,0,0.030266667405764263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,128,0,1,float16,float16,0,0.02388266722361247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,128,0,1,float16,fp8,0,0.02401600033044815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,128,0,1,fp8,fp8,0,0.022474666436513264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,128,0,1,float16,float16,0,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,128,0,1,float16,fp8,0,0.021877333521842957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,128,0,1,fp8,fp8,0,0.02089066555102666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,128,0,1,float16,float16,0,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,128,0,1,float16,fp8,0,0.021925332645575207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,128,0,1,fp8,fp8,0,0.02056533346573512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,128,0,1,float16,float16,0,0.022298666338125866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,128,0,1,float16,fp8,0,0.02288000037272771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,128,0,1,fp8,fp8,0,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,128,0,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,128,0,1,float16,float16,0,0.02237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,128,0,1,float16,fp8,0,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,128,0,1,fp8,fp8,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,128,0,1,float16,float16,0,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,128,0,1,float16,fp8,0,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,128,0,1,float16,float16,0,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,128,0,1,float16,fp8,0,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,128,0,1,fp8,fp8,0,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,128,0,1,float16,float16,0,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,128,0,1,float16,fp8,0,0.018058666338523228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,128,0,1,fp8,fp8,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,128,0,1,float16,float16,0,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,128,0,1,float16,fp8,0,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,128,0,1,fp8,fp8,0,0.0183999997874101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,128,0,1,float16,float16,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,128,0,1,float16,fp8,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,128,0,1,fp8,fp8,0,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,128,0,1,float16,float16,0,0.699669361114502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,128,0,1,float16,fp8,0,0.7015519936879476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,128,0,1,fp8,fp8,0,0.7538666725158691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,128,0,1,float16,float16,0,0.7087519963582357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,128,0,1,float16,fp8,0,0.7035840352376302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,128,0,1,fp8,fp8,0,0.7629013061523438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,128,0,1,float16,float16,0,0.7070133686065674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,128,0,1,float16,fp8,0,0.7111732959747314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,128,0,1,fp8,fp8,0,0.7813440163930258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,128,0,1,float16,float16,0,0.7270613511403402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,128,0,1,float16,fp8,0,0.7233866850535074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,128,0,1,float16,float16,0,0.513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,128,0,1,fp8,fp8,0,0.809397300084432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,128,0,1,float16,fp8,0,0.4956800142923991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,128,0,1,float16,float16,0,0.3516213496526082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,128,0,1,fp8,fp8,0,0.4978239933649699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,128,0,1,float16,fp8,0,0.35277867317199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,128,0,1,fp8,fp8,0,0.38761067390441895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,128,0,1,float16,float16,0,0.3543253342310588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,128,0,1,float16,fp8,0,0.35683735211690265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,128,0,1,fp8,fp8,0,0.3882240056991577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,128,0,1,float16,float16,0,0.35894401868184406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,128,0,1,float16,fp8,0,0.36086400349934894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,128,0,1,fp8,fp8,0,0.3906453450520833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,128,0,1,float16,float16,0,0.37059199810028076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,128,0,1,float16,fp8,0,0.3687093257904053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,128,0,1,fp8,fp8,0,0.39684800306955975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,128,0,1,fp8,fp8,0,0.2593546708424886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,128,0,1,float16,float16,0,0.26772799094518024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,128,0,1,float16,fp8,0,0.25880000988642377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,128,0,1,float16,float16,0,0.1779093345006307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,128,0,1,float16,fp8,0,0.1784906586011251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,128,0,1,fp8,fp8,0,0.20245865980784097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,128,0,1,float16,fp8,0,0.18196266889572144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,128,0,1,float16,float16,0,0.18065067132314047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,128,0,1,fp8,fp8,0,0.2020853360493978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,128,0,1,float16,float16,0,0.1849493384361267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,128,0,1,float16,fp8,0,0.19287999471028647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,128,0,1,float16,fp8,0,0.18420799573262533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,128,0,1,fp8,fp8,0,0.20305599768956503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,128,0,1,fp8,fp8,0,0.14072533448537192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,128,0,1,float16,float16,0,0.1939786672592163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,128,0,1,fp8,fp8,0,0.20709333817164102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,128,0,1,float16,float16,0,0.14296533664067587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,128,0,1,float16,fp8,0,0.13831999897956848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,128,0,1,float16,float16,0,0.09636800487836202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,128,0,1,float16,fp8,0,0.09658132990201314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,128,0,1,fp8,fp8,0,0.11246400078137715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,128,0,1,float16,float16,0,0.09843200445175171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,128,0,1,float16,fp8,0,0.09874666730562846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,128,0,1,fp8,fp8,0,0.11148800452550252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,128,0,1,float16,float16,0,0.09841066598892212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,128,0,1,float16,fp8,0,0.09870933492978413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,128,0,1,fp8,fp8,0,0.11235200365384419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,128,0,1,float16,float16,0,0.10237866640090942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,128,0,1,float16,fp8,0,0.10223999619483948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,128,0,1,fp8,fp8,0,0.11475200454394023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,128,0,1,float16,float16,0,0.08032000064849854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,128,0,1,float16,fp8,0,0.0788266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,128,0,1,fp8,fp8,0,0.07821866869926453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,128,0,1,float16,float16,0,0.05499200026194254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,128,0,1,float16,fp8,0,0.05570666491985321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,128,0,1,fp8,fp8,0,0.061792001128196716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,128,0,1,float16,float16,0,0.05522133409976959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,128,0,1,float16,fp8,0,0.05515199899673462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,128,0,1,fp8,fp8,0,0.06206400195757548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,128,0,1,float16,float16,0,0.055770665407180786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,128,0,1,float16,fp8,0,0.055888002117474876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,128,0,1,fp8,fp8,0,0.06314133107662201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,128,0,1,float16,float16,0,0.05762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,128,0,1,float16,fp8,0,0.056890666484832764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,128,0,1,fp8,fp8,0,0.06429333488146464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,128,0,1,float16,float16,0,0.03923733284076055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,128,0,1,float16,fp8,0,0.038149334490299225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,128,0,1,fp8,fp8,0,0.044879997769991554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,128,0,1,float16,float16,0,0.033200000723203026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,128,0,1,float16,fp8,0,0.0332640012105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,128,0,1,fp8,fp8,0,0.03646933287382126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,128,0,1,float16,float16,0,0.03289066751797994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,128,0,1,float16,fp8,0,0.03270400067170461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,128,0,1,fp8,fp8,0,0.03664000084002813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,128,0,1,float16,float16,0,0.0335359995563825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,128,0,1,float16,fp8,0,0.033701332906881966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,128,0,1,fp8,fp8,0,0.03737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,128,0,1,float16,float16,0,0.033941333492596946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,128,0,1,float16,float16,0,0.022645334402720135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,128,0,1,float16,fp8,0,0.034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,128,0,1,fp8,fp8,0,0.038634667793909706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,128,0,1,float16,float16,0,0.024693332612514496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,128,0,1,float16,fp8,0,0.02502399931351344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,128,0,1,fp8,fp8,0,0.026821332673231762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,128,0,1,float16,fp8,0,0.022698665658632915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,128,0,1,fp8,fp8,0,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,128,0,1,fp8,fp8,0,0.024346667031447094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,128,0,1,float16,float16,0,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,128,0,1,float16,fp8,0,0.0229120006163915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,128,0,1,fp8,fp8,0,0.024501333634058636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,128,0,1,float16,float16,0,0.023589332898457844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,128,0,1,float16,fp8,0,0.023408000667889912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,128,0,1,float16,float16,0,0.023408000667889912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,128,0,1,float16,fp8,0,0.023962666591008503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,128,0,1,fp8,fp8,0,0.02481599897146225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,128,0,1,float16,float16,0,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,128,0,1,float16,fp8,0,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,128,0,1,fp8,fp8,0,0.019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,128,0,1,float16,float16,0,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,128,0,1,float16,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,128,0,1,fp8,fp8,0,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,128,0,1,float16,float16,0,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,128,0,1,float16,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,128,0,1,fp8,fp8,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,128,0,1,float16,float16,0,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,128,0,1,float16,fp8,0,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,128,0,1,fp8,fp8,0,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,128,0,1,float16,float16,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,128,0,1,float16,fp8,0,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,128,0,1,fp8,fp8,0,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,128,0,1,float16,float16,0,0.016000000139077503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,128,0,1,float16,fp8,0,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,128,0,1,fp8,fp8,0,0.017893332988023758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,128,0,1,float16,float16,0,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,128,0,1,float16,fp8,0,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,128,0,1,fp8,fp8,0,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,128,0,1,float16,float16,0,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,128,0,1,float16,fp8,0,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,128,0,1,fp8,fp8,0,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,128,0,1,float16,float16,0,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,128,0,1,float16,fp8,0,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,128,0,1,fp8,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,128,0,1,float16,float16,0,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,128,0,1,float16,fp8,0,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,128,0,1,fp8,fp8,0,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,128,0,1,float16,float16,0,0.4832906723022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,128,0,1,float16,fp8,0,0.4811466534932454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,128,0,1,fp8,fp8,0,0.6052639881769816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,128,0,1,float16,float16,0,0.4803040027618408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,128,0,1,float16,fp8,0,0.4836213191350301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,128,0,1,float16,float16,0,0.49001598358154297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,128,0,1,fp8,fp8,0,0.6080906788508097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,128,0,1,float16,fp8,0,0.49274134635925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,128,0,1,fp8,fp8,0,0.6114933490753174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,128,0,1,float16,float16,0,0.5058720111846924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,128,0,1,float16,fp8,0,0.5048960049947103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,128,0,1,float16,float16,0,0.2404373288154602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,128,0,1,float16,fp8,0,0.23863999048868814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,128,0,1,float16,float16,0,0.3367840051651001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,128,0,1,fp8,fp8,0,0.6189493338267008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,128,0,1,float16,fp8,0,0.331989328066508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,128,0,1,fp8,fp8,0,0.3694560130437215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,128,0,1,fp8,fp8,0,0.31272000074386597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,128,0,1,float16,float16,0,0.24329066276550293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,128,0,1,float16,fp8,0,0.24182399113972983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,128,0,1,fp8,fp8,0,0.31202133496602374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,128,0,1,float16,float16,0,0.25007466475168866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,128,0,1,float16,fp8,0,0.24941333134969076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,128,0,1,float16,float16,0,0.26019734144210815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,128,0,1,fp8,fp8,0,0.3164373238881429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,128,0,1,float16,fp8,0,0.25982399781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,128,0,1,float16,float16,0,0.17460266749064127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,128,0,1,fp8,fp8,0,0.31572266419728595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,128,0,1,float16,fp8,0,0.12593066692352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,128,0,1,float16,fp8,0,0.1709280014038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,128,0,1,float16,float16,0,0.1264746685822805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,128,0,1,fp8,fp8,0,0.19129600127538046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,128,0,1,fp8,fp8,0,0.16563733418782553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,128,0,1,float16,float16,0,0.12707199652989706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,128,0,1,float16,fp8,0,0.1290186643600464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,128,0,1,float16,fp8,0,0.12749333182970682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,128,0,1,fp8,fp8,0,0.1657493313153585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,128,0,1,float16,float16,0,0.12868799765904745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,128,0,1,float16,float16,0,0.1341866652170817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,128,0,1,fp8,fp8,0,0.10549867153167725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,128,0,1,float16,fp8,0,0.06940799951553345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,128,0,1,fp8,fp8,0,0.16877333323160806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,128,0,1,float16,fp8,0,0.13566933075586954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,128,0,1,float16,float16,0,0.09424533446629842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,128,0,1,float16,fp8,0,0.07098666826883952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,128,0,1,fp8,fp8,0,0.16769067446390787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,128,0,1,float16,fp8,0,0.09313066800435384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,128,0,1,float16,float16,0,0.0694400022427241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,128,0,1,fp8,fp8,0,0.09006933371225993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,128,0,1,float16,float16,0,0.07014399766921997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,128,0,1,fp8,fp8,0,0.08869866530100505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,128,0,1,float16,fp8,0,0.0710399995247523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,128,0,1,float16,float16,0,0.07036266724268596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,128,0,1,float16,fp8,0,0.046336000164349876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,128,0,1,fp8,fp8,0,0.0897813340028127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,128,0,1,float16,float16,0,0.07090133428573608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,128,0,1,float16,fp8,0,0.07147733370463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,128,0,1,fp8,fp8,0,0.09210667014122009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,128,0,1,float16,float16,0,0.04780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,128,0,1,fp8,fp8,0,0.058703998724619545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,128,0,1,float16,float16,0,0.040405333042144775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,128,0,1,float16,fp8,0,0.03987200061480204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,128,0,1,fp8,fp8,0,0.05012266834576925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,128,0,1,float16,float16,0,0.03986666599909464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,128,0,1,float16,fp8,0,0.04020266731580099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,128,0,1,fp8,fp8,0,0.05073066552480062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,128,0,1,float16,float16,0,0.04058666775623957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,128,0,1,float16,fp8,0,0.04124266654253006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,128,0,1,fp8,fp8,0,0.05180266499519348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,128,0,1,float16,float16,0,0.040448000033696495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,128,0,1,float16,fp8,0,0.040720000863075256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,128,0,1,fp8,fp8,0,0.052255998055140175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,128,0,1,float16,float16,0,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,128,0,1,float16,fp8,0,0.027999999622503918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,128,0,1,fp8,fp8,0,0.03469333300987879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,128,0,1,float16,float16,0,0.0262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,128,0,1,float16,fp8,0,0.025834667185942333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,128,0,1,fp8,fp8,0,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,128,0,1,float16,float16,0,0.02610666553179423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,128,0,1,float16,fp8,0,0.02595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,128,0,1,fp8,fp8,0,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,128,0,1,float16,fp8,0,0.027045334378878277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,128,0,1,float16,float16,0,0.026799999177455902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,128,0,1,fp8,fp8,0,0.03233599911133448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,128,0,1,float16,float16,0,0.026752000053723652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,128,0,1,float16,fp8,0,0.026837334036827087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,128,0,1,fp8,fp8,0,0.03271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,128,0,1,float16,float16,0,0.01956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,128,0,1,float16,fp8,0,0.020010666300853092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,128,0,1,fp8,fp8,0,0.02327466756105423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,128,0,1,float16,float16,0,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,128,0,1,float16,fp8,0,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,128,0,1,fp8,fp8,0,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,128,0,1,float16,float16,0,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,128,0,1,float16,fp8,0,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,128,0,1,fp8,fp8,0,0.022133332987626392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,128,0,1,float16,float16,0,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,128,0,1,float16,fp8,0,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,128,0,1,fp8,fp8,0,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,128,0,1,float16,float16,0,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,128,0,1,float16,fp8,0,0.01974933346112569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,128,0,1,fp8,fp8,0,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,128,0,1,float16,float16,0,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,128,0,1,float16,fp8,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,128,0,1,fp8,fp8,0,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,128,0,1,float16,float16,0,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,128,0,1,float16,fp8,0,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,128,0,1,fp8,fp8,0,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,128,0,1,float16,float16,0,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,128,0,1,float16,fp8,0,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,128,0,1,fp8,fp8,0,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,128,0,1,float16,float16,0,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,128,0,1,float16,fp8,0,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,128,0,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,128,0,1,float16,float16,0,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,128,0,1,float16,fp8,0,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,128,0,1,fp8,fp8,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,128,0,1,float16,float16,0,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,128,0,1,float16,fp8,0,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,128,0,1,fp8,fp8,0,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,128,0,1,float16,float16,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,128,0,1,float16,fp8,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,128,0,1,fp8,fp8,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,128,0,1,float16,float16,0,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,128,0,1,float16,fp8,0,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,128,0,1,fp8,fp8,0,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,128,0,1,float16,fp8,0,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,128,0,1,float16,float16,0,0.014442666123310724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,128,0,1,float16,fp8,0,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,128,0,1,fp8,fp8,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,128,0,1,float16,float16,0,0.014309333016475042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,128,0,1,fp8,fp8,0,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,128,0,1,float16,float16,0,0.381386677424113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,128,0,1,float16,fp8,0,0.3856906493504842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,128,0,1,float16,float16,0,0.3851253191630046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,128,0,1,fp8,fp8,0,0.5323253472646078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,128,0,1,float16,fp8,0,0.38519465923309326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,128,0,1,float16,float16,0,0.39264531930287677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,128,0,1,fp8,fp8,0,0.5329813162485758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,128,0,1,float16,fp8,0,0.39246400197347003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,128,0,1,fp8,fp8,0,0.5375413497289022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,128,0,1,float16,float16,0,0.40618133544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,128,0,1,float16,fp8,0,0.402672012646993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,128,0,1,float16,float16,0,0.26182933648427326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,128,0,1,fp8,fp8,0,0.5371306737263998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,128,0,1,float16,fp8,0,0.2603519956270854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,128,0,1,float16,float16,0,0.19785600900650024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,128,0,1,fp8,fp8,0,0.30152533451716107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,128,0,1,float16,fp8,0,0.19801600774129233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,128,0,1,float16,float16,0,0.20036266247431436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,128,0,1,float16,fp8,0,0.20038400093714395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,128,0,1,fp8,fp8,0,0.27611732482910156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,128,0,1,fp8,fp8,0,0.2764906684557597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,128,0,1,float16,float16,0,0.20244266589482626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,128,0,1,float16,fp8,0,0.20150933663050333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,128,0,1,fp8,fp8,0,0.2769813338915507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,128,0,1,float16,float16,0,0.20736533403396606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,128,0,1,float16,fp8,0,0.2076746622721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,128,0,1,float16,float16,0,0.12993599971135458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,128,0,1,fp8,fp8,0,0.27779199679692584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,128,0,1,float16,fp8,0,0.12761066357294717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,128,0,1,fp8,fp8,0,0.1600213348865509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,128,0,1,float16,float16,0,0.10500267148017883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,128,0,1,float16,fp8,0,0.10585066676139832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,128,0,1,fp8,fp8,0,0.14415466785430908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,128,0,1,float16,float16,0,0.10458133618036906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,128,0,1,float16,fp8,0,0.10598400235176086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,128,0,1,fp8,fp8,0,0.1453546682993571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,128,0,1,float16,float16,0,0.10542399684588115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,128,0,1,float16,fp8,0,0.10548266768455505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,128,0,1,float16,fp8,0,0.06403199831644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,128,0,1,fp8,fp8,0,0.14538133144378662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,128,0,1,float16,float16,0,0.10661333799362183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,128,0,1,float16,fp8,0,0.10639466842015584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,128,0,1,fp8,fp8,0,0.14617600043614706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,128,0,1,float16,float16,0,0.06577600042025249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,128,0,1,float16,float16,0,0.058592001597086586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,128,0,1,fp8,fp8,0,0.08680533369382222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,128,0,1,float16,float16,0,0.05820799867312113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,128,0,1,float16,fp8,0,0.05914666752020518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,128,0,1,fp8,fp8,0,0.07845333218574524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,128,0,1,fp8,fp8,0,0.07979199786980946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,128,0,1,float16,fp8,0,0.05850133299827576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,128,0,1,fp8,fp8,0,0.07866666714350383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,128,0,1,float16,float16,0,0.05895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,128,0,1,float16,fp8,0,0.05933333436648051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,128,0,1,float16,float16,0,0.058917333682378135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,128,0,1,float16,fp8,0,0.0594400018453598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,128,0,1,fp8,fp8,0,0.08030933141708374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,128,0,1,float16,fp8,0,0.035631999373435974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,128,0,1,float16,float16,0,0.03698666642109553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,128,0,1,float16,fp8,0,0.037231999138991036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,128,0,1,float16,float16,0,0.03581333408753077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,128,0,1,fp8,fp8,0,0.04854399959246317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,128,0,1,float16,float16,0,0.03523733218510946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,128,0,1,fp8,fp8,0,0.045850664377212524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,128,0,1,float16,fp8,0,0.03668266783157984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,128,0,1,fp8,fp8,0,0.04680533210436503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,128,0,1,float16,float16,0,0.03589866558710734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,128,0,1,fp8,fp8,0,0.04743466774622599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,128,0,1,float16,fp8,0,0.035786665976047516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,128,0,1,fp8,fp8,0,0.047610665361086525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,128,0,1,float16,float16,0,0.036757332583268486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,128,0,1,float16,fp8,0,0.036357333262761436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,128,0,1,float16,float16,0,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,128,0,1,float16,fp8,0,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,128,0,1,fp8,fp8,0,0.031040000418821972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,128,0,1,float16,float16,0,0.02390933285156886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,128,0,1,float16,fp8,0,0.02384000023206075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,128,0,1,fp8,fp8,0,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,128,0,1,float16,float16,0,0.02443733314673106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,128,0,1,float16,fp8,0,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,128,0,1,fp8,fp8,0,0.029845332105954487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,128,0,1,float16,float16,0,0.02489600082238515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,128,0,1,float16,fp8,0,0.024421334266662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,128,0,1,fp8,fp8,0,0.029701332251230877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,128,0,1,float16,float16,0,0.024362665911515553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,128,0,1,float16,fp8,0,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,128,0,1,fp8,fp8,0,0.030085332691669464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,128,0,1,float16,float16,0,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,128,0,1,float16,fp8,0,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,128,0,1,fp8,fp8,0,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,128,0,1,float16,float16,0,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,128,0,1,float16,fp8,0,0.018288000176350277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,128,0,1,fp8,fp8,0,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,128,0,1,float16,float16,0,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,128,0,1,float16,fp8,0,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,128,0,1,fp8,fp8,0,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,128,0,1,float16,float16,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,128,0,1,float16,fp8,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,128,0,1,fp8,fp8,0,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,128,0,1,float16,float16,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,128,0,1,float16,fp8,0,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,128,0,1,fp8,fp8,0,0.021749332547187805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,128,0,1,float16,float16,0,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,128,0,1,float16,fp8,0,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,128,0,1,fp8,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,128,0,1,float16,float16,0,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,128,0,1,float16,fp8,0,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,128,0,1,fp8,fp8,0,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,128,0,1,float16,float16,0,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,128,0,1,float16,fp8,0,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,128,0,1,fp8,fp8,0,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,128,0,1,fp8,fp8,0,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,128,0,1,float16,float16,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,128,0,1,float16,fp8,0,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,128,0,1,fp8,fp8,0,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,128,0,1,float16,float16,0,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,128,0,1,float16,fp8,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,128,0,1,float16,float16,0,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,128,0,1,float16,fp8,0,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,128,0,1,float16,fp8,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,128,0,1,fp8,fp8,0,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,128,0,1,float16,float16,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,128,0,1,float16,fp8,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,128,0,1,fp8,fp8,0,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,128,0,1,float16,float16,0,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,128,0,1,fp8,fp8,0,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,128,0,1,float16,float16,0,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,128,0,1,float16,fp8,0,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,128,0,1,fp8,fp8,0,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,128,0,1,float16,float16,0,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,128,0,1,float16,fp8,0,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,128,0,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,128,0,1,fp8,fp8,0,34.41684214274088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,128,0,1,fp8,fp8,0,34.71040598551432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,128,0,1,float16,float16,0,50.48180643717448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,128,0,1,float16,fp8,0,50.7035166422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,128,0,1,float16,float16,0,49.96552530924479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,128,0,1,float16,fp8,0,50.937825520833336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,128,0,1,float16,fp8,0,49.9475351969401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,128,0,1,float16,float16,0,51.31488037109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,128,0,1,fp8,fp8,0,33.46756235758463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,128,0,1,fp8,fp8,0,18.12982432047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,128,0,1,float16,float16,0,28.13105010986328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,128,0,1,float16,fp8,0,28.56946563720703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,128,0,1,float16,float16,0,28.027503967285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,128,0,1,fp8,fp8,0,34.33010609944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,128,0,1,float16,float16,0,52.716756184895836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,128,0,1,fp8,fp8,0,16.805899302164715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,128,0,1,float16,fp8,0,24.99334971110026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,128,0,1,fp8,fp8,0,16.637322743733723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,128,0,1,float16,fp8,0,50.293172200520836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,128,0,1,float16,float16,0,25.101712544759113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,128,0,1,float16,fp8,0,25.91802215576172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,128,0,1,float16,float16,0,26.590794881184895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,128,0,1,float16,fp8,0,27.02137502034505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,128,0,1,fp8,fp8,0,18.12288538614909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,128,0,1,float16,float16,0,13.812175750732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,128,0,1,float16,fp8,0,13.813861846923828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,128,0,1,fp8,fp8,0,17.310731252034504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,128,0,1,fp8,fp8,0,9.438016255696615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,128,0,1,float16,float16,0,25.51256052652995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,128,0,1,float16,fp8,0,12.763844807942709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,128,0,1,float16,fp8,0,27.779151916503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,128,0,1,float16,float16,0,12.853764851888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,128,0,1,fp8,fp8,0,8.512704213460287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,128,0,1,fp8,fp8,0,8.396165211995443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,128,0,1,float16,float16,0,12.721248626708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,128,0,1,float16,fp8,0,13.338138580322266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,128,0,1,float16,float16,0,12.671855926513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,128,0,1,fp8,fp8,0,8.44546635945638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,128,0,1,fp8,fp8,0,8.45248031616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,128,0,1,float16,float16,0,6.91107177734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,128,0,1,float16,fp8,0,12.918020884195963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,128,0,1,fp8,fp8,0,4.634981473286946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,128,0,1,float16,fp8,0,12.861541748046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,128,0,1,float16,float16,0,12.82436752319336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,128,0,1,float16,fp8,0,7.002570470174153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,128,0,1,float16,float16,0,6.460639953613281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,128,0,1,fp8,fp8,0,4.4557491938273115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,128,0,1,fp8,fp8,0,4.648458798726399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,128,0,1,float16,fp8,0,6.802309036254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,128,0,1,float16,float16,0,6.999029159545898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,128,0,1,float16,fp8,0,6.828234354654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,128,0,1,float16,float16,0,6.945738474527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,128,0,1,fp8,fp8,0,4.329642613728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,128,0,1,float16,fp8,0,6.7050825754801435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,128,0,1,fp8,fp8,0,4.477941195170085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,128,0,1,float16,float16,0,6.968458811442058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,128,0,1,float16,fp8,0,7.261599858601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,128,0,1,fp8,fp8,0,19.848927815755207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,128,0,1,fp8,fp8,0,19.617834726969402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,128,0,1,float16,float16,0,29.47730255126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,128,0,1,float16,fp8,0,29.285675048828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,128,0,1,float16,float16,0,28.8450444539388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,128,0,1,float16,fp8,0,30.24353535970052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,128,0,1,float16,fp8,0,30.019119262695312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,128,0,1,float16,float16,0,30.364415486653645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,128,0,1,fp8,fp8,0,20.53760019938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,128,0,1,fp8,fp8,0,10.547157287597656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,128,0,1,float16,float16,0,15.656667073567709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,128,0,1,float16,fp8,0,15.184112548828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,128,0,1,fp8,fp8,0,19.80780792236328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,128,0,1,float16,float16,0,15.220437367757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,128,0,1,float16,float16,0,29.64312998453776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,128,0,1,fp8,fp8,0,9.67198371887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,128,0,1,float16,fp8,0,15.026196797688803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,128,0,1,float16,fp8,0,29.33148193359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,128,0,1,fp8,fp8,0,9.981450398763021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,128,0,1,float16,fp8,0,14.940811157226562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,128,0,1,float16,float16,0,14.600410461425781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,128,0,1,float16,float16,0,15.150335947672525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,128,0,1,fp8,fp8,0,10.061434427897135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,128,0,1,float16,fp8,0,14.662698109944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,128,0,1,float16,float16,0,7.941392262776692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,128,0,1,float16,fp8,0,7.513914744059245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,128,0,1,fp8,fp8,0,10.121429443359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,128,0,1,fp8,fp8,0,5.466559727986653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,128,0,1,float16,float16,0,15.067002614339193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,128,0,1,float16,fp8,0,15.108304341634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,128,0,1,float16,float16,0,7.811679840087891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,128,0,1,fp8,fp8,0,5.089701334635417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,128,0,1,fp8,fp8,0,5.278106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,128,0,1,float16,fp8,0,7.597279866536458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,128,0,1,float16,float16,0,7.316656112670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,128,0,1,float16,fp8,0,7.604474385579427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,128,0,1,float16,float16,0,7.740613301595052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,128,0,1,fp8,fp8,0,5.23089059193929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,128,0,1,float16,fp8,0,7.536266962687175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,128,0,1,float16,float16,0,3.9911626180013022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,128,0,1,float16,fp8,0,4.053882598876953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,128,0,1,fp8,fp8,0,5.146575927734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,128,0,1,fp8,fp8,0,2.7561972935994468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,128,0,1,float16,float16,0,7.453290939331055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,128,0,1,float16,float16,0,3.875621477762858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,128,0,1,float16,fp8,0,7.9010664621988935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,128,0,1,fp8,fp8,0,2.7083040873209634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,128,0,1,fp8,fp8,0,2.590341409047445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,128,0,1,float16,fp8,0,3.808261235555013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,128,0,1,float16,float16,0,3.813082695007324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,128,0,1,float16,fp8,0,3.934650739034017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,128,0,1,float16,float16,0,3.7885707219441733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,128,0,1,fp8,fp8,0,2.8358081181844077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,128,0,1,float16,fp8,0,3.8474559783935547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,128,0,1,fp8,fp8,0,2.613706588745117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,128,0,1,float16,float16,0,4.030341466267903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,128,0,1,float16,fp8,0,4.0372053782145185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,128,0,1,fp8,fp8,0,14.445808410644531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,128,0,1,fp8,fp8,0,14.123434702555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,128,0,1,float16,float16,0,20.990890502929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,128,0,1,float16,fp8,0,20.633562723795574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,128,0,1,float16,float16,0,22.264597574869793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,128,0,1,float16,fp8,0,21.291851043701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,128,0,1,float16,float16,0,21.414382934570312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,128,0,1,float16,fp8,0,20.1421381632487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,128,0,1,fp8,fp8,0,14.906239827473959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,128,0,1,fp8,fp8,0,7.602506637573242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,128,0,1,float16,float16,0,10.997435251871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,128,0,1,float16,fp8,0,10.886868794759115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,128,0,1,fp8,fp8,0,14.387339274088541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,128,0,1,float16,float16,0,21.84064483642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,128,0,1,fp8,fp8,0,7.10540771484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,128,0,1,float16,float16,0,12.274336496988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,128,0,1,float16,fp8,0,10.488405227661133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,128,0,1,float16,fp8,0,20.884063720703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,128,0,1,fp8,fp8,0,7.228858947753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,128,0,1,float16,float16,0,10.276320139567057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,128,0,1,float16,fp8,0,10.475178400675455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,128,0,1,fp8,fp8,0,7.187119801839192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,128,0,1,float16,float16,0,10.450074513753256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,128,0,1,float16,float16,0,5.459248224894206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,128,0,1,fp8,fp8,0,7.117839813232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,128,0,1,float16,fp8,0,10.427973429361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,128,0,1,float16,fp8,0,5.568597157796224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,128,0,1,float16,float16,0,10.661984125773111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,128,0,1,float16,fp8,0,10.537061055501303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,128,0,1,fp8,fp8,0,3.7012160619099936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,128,0,1,fp8,fp8,0,3.6750561396280923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,128,0,1,float16,float16,0,5.308783849080403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,128,0,1,float16,fp8,0,5.583994547526042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,128,0,1,fp8,fp8,0,3.6647841135660806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,128,0,1,float16,float16,0,5.3354237874348955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,128,0,1,float16,fp8,0,5.38107172648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,128,0,1,fp8,fp8,0,3.6722774505615234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,128,0,1,float16,float16,0,5.335034688313802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,128,0,1,float16,fp8,0,5.2445065180460615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,128,0,1,float16,float16,0,2.9020907084147134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,128,0,1,float16,fp8,0,2.866922696431478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,128,0,1,fp8,fp8,0,3.744629224141439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,128,0,1,fp8,fp8,0,1.8809119860331218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,128,0,1,float16,float16,0,5.232138633728027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,128,0,1,float16,fp8,0,5.424581527709961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,128,0,1,float16,float16,0,2.533418655395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,128,0,1,float16,fp8,0,2.6734933853149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,128,0,1,fp8,fp8,0,1.8951306343078613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,128,0,1,fp8,fp8,0,1.8196372985839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,128,0,1,float16,float16,0,2.7269118626912436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,128,0,1,float16,fp8,0,2.9347734451293945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,128,0,1,float16,float16,0,2.795962651570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,128,0,1,fp8,fp8,0,1.879269282023112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,128,0,1,float16,fp8,0,2.698282559712728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,128,0,1,fp8,fp8,0,1.836997350056966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,128,0,1,float16,float16,0,2.730976104736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,128,0,1,float16,fp8,0,2.8019841512044272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,128,0,1,fp8,fp8,0,18.78306706746419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,128,0,1,fp8,fp8,0,18.773946126302082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,128,0,1,float16,float16,0,27.744394938151043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,128,0,1,float16,float16,0,27.743024190266926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,128,0,1,float16,fp8,0,28.38905080159505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,128,0,1,float16,fp8,0,27.897504170735676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,128,0,1,float16,float16,0,27.304219563802082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,128,0,1,float16,fp8,0,27.9474614461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,128,0,1,fp8,fp8,0,18.949637095133465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,128,0,1,fp8,fp8,0,9.989445368448893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,128,0,1,float16,float16,0,14.835413614908854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,128,0,1,float16,fp8,0,14.433055877685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,128,0,1,float16,float16,0,13.857205708821615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,128,0,1,fp8,fp8,0,18.873279571533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,128,0,1,float16,float16,0,27.852218627929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,128,0,1,fp8,fp8,0,9.58786646525065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,128,0,1,float16,fp8,0,13.959055582682291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,128,0,1,fp8,fp8,0,9.536144256591797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,128,0,1,float16,fp8,0,27.58789825439453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,128,0,1,float16,fp8,0,13.647701263427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,128,0,1,float16,float16,0,13.997083028157553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,128,0,1,fp8,fp8,0,9.620831807454428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,128,0,1,float16,float16,0,13.9508056640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,128,0,1,float16,fp8,0,13.7848269144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,128,0,1,fp8,fp8,0,9.671365102132162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,128,0,1,fp8,fp8,0,5.151589393615723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,128,0,1,float16,float16,0,7.398901621500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,128,0,1,float16,fp8,0,8.015381495157877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,128,0,1,float16,float16,0,13.888800303141275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,128,0,1,float16,fp8,0,14.47756830851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,128,0,1,float16,float16,0,7.121695836385091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,128,0,1,float16,fp8,0,6.892927805582683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,128,0,1,fp8,fp8,0,4.802650769551595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,128,0,1,fp8,fp8,0,4.4202985763549805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,128,0,1,float16,float16,0,6.959456125895183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,128,0,1,float16,fp8,0,7.062074661254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,128,0,1,fp8,fp8,0,4.709199905395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,128,0,1,float16,float16,0,6.988597234090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,128,0,1,float16,fp8,0,6.967946370442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,128,0,1,fp8,fp8,0,4.735589345296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,128,0,1,float16,float16,0,3.613386789957682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,128,0,1,float16,fp8,0,3.690021197001139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,128,0,1,fp8,fp8,0,2.508416016896566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,128,0,1,float16,float16,0,7.007274627685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,128,0,1,float16,fp8,0,6.945056279500325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,128,0,1,fp8,fp8,0,2.3574026425679526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,128,0,1,float16,float16,0,3.531466801961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,128,0,1,fp8,fp8,0,2.359663963317871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,128,0,1,float16,fp8,0,3.8950719833374023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,128,0,1,float16,float16,0,3.5038026173909507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,128,0,1,float16,fp8,0,3.5123465855916343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,128,0,1,float16,float16,0,3.4998559951782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,128,0,1,fp8,fp8,0,2.406442642211914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,128,0,1,float16,fp8,0,3.5290346145629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,128,0,1,float16,float16,0,1.7706987063090007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,128,0,1,fp8,fp8,0,2.303237279256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,128,0,1,float16,float16,0,3.6904214223225913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,128,0,1,float16,fp8,0,1.8496160507202148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,128,0,1,fp8,fp8,0,1.2897066275278728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,128,0,1,float16,fp8,0,3.489898681640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,128,0,1,fp8,fp8,0,1.2386613686879475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,128,0,1,float16,fp8,0,1.7401280403137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,128,0,1,float16,float16,0,1.8438080151875813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,128,0,1,float16,float16,0,1.776309331258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,128,0,1,fp8,fp8,0,1.2678933143615723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,128,0,1,float16,fp8,0,1.7093706130981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,128,0,1,fp8,fp8,0,1.3423360188802083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,128,0,1,float16,float16,0,1.7336959838867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,128,0,1,float16,fp8,0,1.737178643544515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,128,0,1,float16,float16,0,1.7840266227722168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,128,0,1,fp8,fp8,0,1.2375893592834473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,128,0,1,float16,fp8,0,1.8045120239257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,128,0,1,fp8,fp8,0,11.148293813069662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,128,0,1,fp8,fp8,0,11.283828735351562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,128,0,1,float16,float16,0,16.34702428181966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,128,0,1,float16,fp8,0,16.51410166422526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,128,0,1,float16,float16,0,16.581631978352863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,128,0,1,float16,fp8,0,16.31606928507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,128,0,1,float16,float16,0,16.36291249593099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,128,0,1,float16,fp8,0,15.709152221679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,128,0,1,fp8,fp8,0,11.562751770019531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,128,0,1,fp8,fp8,0,6.0396162668863935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,128,0,1,float16,float16,0,8.647567749023438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,128,0,1,float16,fp8,0,8.612880071004232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,128,0,1,fp8,fp8,0,11.397771199544271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,128,0,1,float16,float16,0,16.72817611694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,128,0,1,float16,float16,0,7.855690638224284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,128,0,1,fp8,fp8,0,5.532911936442058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,128,0,1,float16,fp8,0,8.50323740641276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,128,0,1,float16,fp8,0,16.61950429280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,128,0,1,fp8,fp8,0,5.8191680908203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,128,0,1,float16,float16,0,8.12674649556478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,128,0,1,float16,fp8,0,8.263856252034506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,128,0,1,fp8,fp8,0,5.784698486328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,128,0,1,float16,float16,0,8.21021842956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,128,0,1,float16,fp8,0,8.410741170247396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,128,0,1,float16,float16,0,4.261925379435222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,128,0,1,float16,fp8,0,4.353957176208496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,128,0,1,fp8,fp8,0,3.013728141784668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,128,0,1,fp8,fp8,0,5.62718391418457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,128,0,1,float16,fp8,0,8.477173487345377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,128,0,1,fp8,fp8,0,2.7619520823160806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,128,0,1,float16,float16,0,8.069087982177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,128,0,1,float16,float16,0,4.046762784322103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,128,0,1,float16,fp8,0,4.910837491353353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,128,0,1,fp8,fp8,0,2.755903879801432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,128,0,1,float16,float16,0,4.113381385803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,128,0,1,float16,fp8,0,4.097920099894206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,128,0,1,fp8,fp8,0,2.7175467809041343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,128,0,1,float16,float16,0,2.1990559895833335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,128,0,1,float16,float16,0,3.910917282104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,128,0,1,float16,fp8,0,3.9908746083577475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,128,0,1,fp8,fp8,0,2.9141012827555337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,128,0,1,float16,fp8,0,2.1017600695292153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,128,0,1,float16,float16,0,4.092485427856445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,128,0,1,fp8,fp8,0,1.5738986333211262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,128,0,1,float16,fp8,0,4.228357315063477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,128,0,1,fp8,fp8,0,1.405786673227946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,128,0,1,float16,float16,0,2.0379573504130044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,128,0,1,float16,fp8,0,2.0870772997538247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,128,0,1,fp8,fp8,0,1.416368007659912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,128,0,1,float16,float16,0,2.038405259450277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,128,0,1,float16,fp8,0,1.9192800521850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,128,0,1,float16,float16,0,2.0282665888468423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,128,0,1,fp8,fp8,0,1.3864320119222004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,128,0,1,float16,fp8,0,2.0057493845621743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,128,0,1,float16,float16,0,1.135317325592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,128,0,1,fp8,fp8,0,1.4140267372131348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,128,0,1,float16,fp8,0,1.0849546591440837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,128,0,1,float16,float16,0,2.058133284250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,128,0,1,fp8,fp8,0,0.8635839621225992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,128,0,1,float16,fp8,0,2.106682618459066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,128,0,1,float16,float16,0,1.0648053487141926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,128,0,1,fp8,fp8,0,0.7905866305033366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,128,0,1,float16,fp8,0,1.1177226702372234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,128,0,1,fp8,fp8,0,0.8078453540802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,128,0,1,float16,float16,0,1.044533332188924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,128,0,1,float16,fp8,0,1.0402666727701824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,128,0,1,float16,float16,0,1.0660693645477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,128,0,1,fp8,fp8,0,0.7808799743652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,128,0,1,float16,fp8,0,1.083194653193156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,128,0,1,float16,float16,0,1.054357369740804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,128,0,1,fp8,fp8,0,0.7982347011566162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,128,0,1,float16,fp8,0,1.082645336786906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,128,0,1,fp8,fp8,0,11.300421396891275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,128,0,1,fp8,fp8,0,11.22921625773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,128,0,1,float16,float16,0,15.900751749674479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,128,0,1,float16,fp8,0,16.331231435139973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,128,0,1,float16,float16,0,16.289866129557293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,128,0,1,float16,float16,0,15.307632446289062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,128,0,1,float16,fp8,0,15.810410817464193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,128,0,1,fp8,fp8,0,11.188186645507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,128,0,1,fp8,fp8,0,6.127301534016927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,128,0,1,float16,fp8,0,15.174271901448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,128,0,1,float16,float16,0,8.475253423055014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,128,0,1,float16,fp8,0,8.857274373372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,128,0,1,fp8,fp8,0,11.4475949605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,128,0,1,float16,float16,0,15.9289919535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,128,0,1,fp8,fp8,0,5.376074473063151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,128,0,1,float16,float16,0,7.866416295369466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,128,0,1,float16,fp8,0,8.119173049926758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,128,0,1,float16,fp8,0,16.268442789713543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,128,0,1,float16,float16,0,8.106559753417969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,128,0,1,float16,fp8,0,8.340949376424154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,128,0,1,fp8,fp8,0,5.691386540730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,128,0,1,float16,float16,0,8.064608256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,128,0,1,fp8,fp8,0,5.47596804300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,128,0,1,float16,fp8,0,7.63047472635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,128,0,1,float16,float16,0,4.218469301859538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,128,0,1,float16,fp8,0,4.297685305277507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,128,0,1,fp8,fp8,0,5.600165049235026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,128,0,1,float16,float16,0,8.112122853597006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,128,0,1,fp8,fp8,0,3.014944076538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,128,0,1,float16,fp8,0,8.043024063110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,128,0,1,float16,float16,0,3.8767786026000977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,128,0,1,fp8,fp8,0,2.654575983683268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,128,0,1,float16,fp8,0,4.339738527933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,128,0,1,float16,float16,0,3.9359893798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,128,0,1,fp8,fp8,0,2.725056012471517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,128,0,1,float16,fp8,0,3.8172480265299478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,128,0,1,float16,float16,0,3.8842080434163413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,128,0,1,fp8,fp8,0,2.7891785303751626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,128,0,1,float16,fp8,0,3.8877172470092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,128,0,1,float16,float16,0,2.0335466066996255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,128,0,1,float16,float16,0,3.92739200592041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,128,0,1,float16,fp8,0,2.0176587104797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,128,0,1,fp8,fp8,0,1.510874589284261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,128,0,1,fp8,fp8,0,2.7838134765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,128,0,1,float16,float16,0,1.9465227127075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,128,0,1,float16,fp8,0,4.370149294535319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,128,0,1,float16,fp8,0,1.8763893445332844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,128,0,1,fp8,fp8,0,1.3649813334147136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,128,0,1,fp8,fp8,0,1.356053352355957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,128,0,1,float16,float16,0,1.8396533330281575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,128,0,1,float16,fp8,0,1.9372426668802898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,128,0,1,float16,float16,0,1.9596106211344402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,128,0,1,fp8,fp8,0,1.3767733573913574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,128,0,1,float16,fp8,0,1.9721226692199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,128,0,1,float16,float16,0,1.9745599428812664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,128,0,1,float16,fp8,0,2.032800038655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,128,0,1,fp8,fp8,0,1.3524746894836426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,128,0,1,float16,float16,0,1.0245013236999512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,128,0,1,fp8,fp8,0,0.8199093341827393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,128,0,1,float16,fp8,0,1.0594186782836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,128,0,1,float16,float16,0,0.9547466437021891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,128,0,1,fp8,fp8,0,0.7979466915130615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,128,0,1,float16,fp8,0,1.0112853050231934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,128,0,1,float16,fp8,0,0.9584746360778809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,128,0,1,fp8,fp8,0,0.7495520114898682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,128,0,1,float16,float16,0,1.0112373034159343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,128,0,1,float16,float16,0,0.9793333212534586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,128,0,1,fp8,fp8,0,0.7339253425598145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,128,0,1,float16,fp8,0,0.995306650797526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,128,0,1,float16,float16,0,0.989834705988566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,128,0,1,float16,float16,0,0.5608906745910645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,128,0,1,float16,fp8,0,0.5706293185551962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,128,0,1,float16,fp8,0,0.9887519677480062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,128,0,1,fp8,fp8,0,0.7372639973958334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,128,0,1,fp8,fp8,0,0.4379040002822876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,128,0,1,float16,float16,0,0.5406026840209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,128,0,1,float16,fp8,0,0.5501973231633505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,128,0,1,fp8,fp8,0,0.4057600100835164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,128,0,1,float16,float16,0,0.5398559967676798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,128,0,1,fp8,fp8,0,0.40890665849049884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,128,0,1,float16,fp8,0,0.5461653470993042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,128,0,1,float16,float16,0,0.5411626497904459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,128,0,1,fp8,fp8,0,0.40106133619944256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,128,0,1,float16,fp8,0,0.5494186480840048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,128,0,1,float16,float16,0,0.5429386695226034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,128,0,1,fp8,fp8,0,0.4017813205718994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,128,0,1,float16,fp8,0,0.5461920102437338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,128,0,1,fp8,fp8,0,6.796410878499349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,128,0,1,fp8,fp8,0,6.65017573038737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,128,0,1,float16,float16,0,9.266096115112305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,128,0,1,float16,fp8,0,9.32911491394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,128,0,1,float16,float16,0,9.159797032674154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,128,0,1,float16,fp8,0,9.476336161295572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,128,0,1,float16,float16,0,9.346469243367514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,128,0,1,float16,fp8,0,9.431493123372396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,128,0,1,fp8,fp8,0,6.768933614095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,128,0,1,fp8,fp8,0,3.7923574447631836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,128,0,1,float16,float16,0,5.051093419392903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,128,0,1,float16,fp8,0,5.219146728515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,128,0,1,fp8,fp8,0,6.881994883219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,128,0,1,float16,float16,0,9.533541361490885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,128,0,1,fp8,fp8,0,3.3457654317220054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,128,0,1,float16,float16,0,4.700122515360515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,128,0,1,float16,fp8,0,9.62709871927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,128,0,1,float16,fp8,0,4.7206878662109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,128,0,1,float16,float16,0,4.776938756306966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,128,0,1,fp8,fp8,0,3.398106575012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,128,0,1,float16,fp8,0,4.765285491943359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,128,0,1,fp8,fp8,0,3.41374937693278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,128,0,1,float16,float16,0,4.900170644124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,128,0,1,float16,float16,0,2.4972586631774902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,128,0,1,float16,fp8,0,4.764080047607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,128,0,1,fp8,fp8,0,3.420618693033854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,128,0,1,float16,float16,0,4.90995724995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,128,0,1,float16,fp8,0,2.7706292470296225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,128,0,1,fp8,fp8,0,1.9195787111918132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,128,0,1,float16,fp8,0,4.850271860758464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,128,0,1,float16,float16,0,2.280773321787516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,128,0,1,float16,fp8,0,2.2675253550211587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,128,0,1,fp8,fp8,0,1.7774933179219563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,128,0,1,float16,float16,0,2.1891892751057944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,128,0,1,fp8,fp8,0,1.714037259419759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,128,0,1,float16,fp8,0,2.2991627057393393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,128,0,1,float16,float16,0,2.289210637410482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,128,0,1,float16,fp8,0,2.4957547187805176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,128,0,1,fp8,fp8,0,1.7361812591552734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,128,0,1,float16,float16,0,1.2610560258229573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,128,0,1,fp8,fp8,0,1.6772640546162922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,128,0,1,float16,fp8,0,1.2491040229797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,128,0,1,float16,float16,0,2.481898625691732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,128,0,1,float16,fp8,0,2.4077812830607095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,128,0,1,fp8,fp8,0,1.0214293003082275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,128,0,1,fp8,fp8,0,0.8740159670511881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,128,0,1,float16,fp8,0,1.1348533630371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,128,0,1,float16,float16,0,1.2388640244801838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,128,0,1,float16,float16,0,1.158901294072469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,128,0,1,fp8,fp8,0,0.9313440322875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,128,0,1,float16,fp8,0,1.20740803082784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,128,0,1,float16,float16,0,1.1601333618164062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,128,0,1,fp8,fp8,0,0.8599519729614258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,128,0,1,float16,fp8,0,1.1651519934336345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,128,0,1,float16,float16,0,0.6589653491973877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,128,0,1,float16,float16,0,1.2544426918029785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,128,0,1,float16,fp8,0,1.263210693995158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,128,0,1,float16,fp8,0,0.7031573454538981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,128,0,1,fp8,fp8,0,0.8951520125071207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,128,0,1,fp8,fp8,0,0.5639520088831583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,128,0,1,float16,float16,0,0.6085120042165121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,128,0,1,float16,fp8,0,0.6646933158238729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,128,0,1,fp8,fp8,0,0.5019199848175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,128,0,1,float16,float16,0,0.6731786727905273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,128,0,1,fp8,fp8,0,0.4978346824645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,128,0,1,float16,fp8,0,0.7016586462656657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,128,0,1,float16,float16,0,0.6070186694463094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,128,0,1,fp8,fp8,0,0.4931306838989258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,128,0,1,float16,fp8,0,0.6483360131581625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,128,0,1,float16,float16,0,0.6455413500467936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,128,0,1,float16,fp8,0,0.6474666595458984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,128,0,1,float16,float16,0,0.37138132254282635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,128,0,1,float16,fp8,0,0.3980213403701782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,128,0,1,float16,float16,0,0.36132268110911053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,128,0,1,fp8,fp8,0,0.4997440179189046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,128,0,1,fp8,fp8,0,0.29082133372624713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,128,0,1,float16,fp8,0,0.3739573160807292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,128,0,1,fp8,fp8,0,0.2714293400446574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,128,0,1,float16,float16,0,0.3689599831899007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,128,0,1,fp8,fp8,0,0.2747039993604024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,128,0,1,float16,fp8,0,0.3550453186035156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,128,0,1,float16,float16,0,0.3641066551208496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,128,0,1,float16,fp8,0,0.3526933193206787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,128,0,1,fp8,fp8,0,0.2664586702982585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,128,0,1,float16,float16,0,0.35838933785756427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,128,0,1,float16,fp8,0,0.3599253495534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,128,0,1,fp8,fp8,0,0.26843732595443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,128,0,1,fp8,fp8,0,7.1528746287028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,128,0,1,fp8,fp8,0,7.242015838623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,128,0,1,float16,float16,0,9.559082667032877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,128,0,1,float16,fp8,0,9.529424031575521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,128,0,1,float16,float16,0,9.53718376159668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,128,0,1,float16,fp8,0,9.481002807617188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,128,0,1,float16,float16,0,9.530415852864584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,128,0,1,float16,fp8,0,9.454885482788086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,128,0,1,fp8,fp8,0,4.118330637613933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,128,0,1,fp8,fp8,0,7.376549402872722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,128,0,1,float16,float16,0,4.744277318318685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,128,0,1,float16,float16,0,5.247418721516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,128,0,1,float16,fp8,0,5.340090433756511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,128,0,1,fp8,fp8,0,7.367114384969075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,128,0,1,float16,float16,0,9.725296020507812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,128,0,1,fp8,fp8,0,3.52840518951416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,128,0,1,fp8,fp8,0,3.5825173060099282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,128,0,1,float16,fp8,0,9.736181259155273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,128,0,1,float16,fp8,0,4.791130701700847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,128,0,1,float16,float16,0,4.68228276570638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,128,0,1,float16,fp8,0,4.642965316772461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,128,0,1,float16,float16,0,4.919237454732259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,128,0,1,float16,fp8,0,4.732197443644206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,128,0,1,fp8,fp8,0,3.6331307093302407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,128,0,1,float16,float16,0,2.590655962626139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,128,0,1,fp8,fp8,0,3.6319519678751626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,128,0,1,float16,float16,0,4.922608057657878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,128,0,1,float16,fp8,0,4.911984125773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,128,0,1,fp8,fp8,0,2.026282628377279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,128,0,1,float16,float16,0,2.310863971710205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,128,0,1,float16,fp8,0,2.613861401875814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,128,0,1,fp8,fp8,0,1.7422240575154622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,128,0,1,float16,fp8,0,2.231839974721273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,128,0,1,fp8,fp8,0,1.7344427108764648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,128,0,1,float16,fp8,0,2.2545493443806968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,128,0,1,float16,float16,0,2.2561705907185874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,128,0,1,float16,float16,0,2.326709270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,128,0,1,float16,fp8,0,2.2922399838765464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,128,0,1,fp8,fp8,0,1.7456159591674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,128,0,1,float16,float16,0,2.3154239654541016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,128,0,1,float16,float16,0,1.2977386315663655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,128,0,1,fp8,fp8,0,1.1160746415456135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,128,0,1,float16,fp8,0,1.3069120248158772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,128,0,1,float16,float16,0,1.1058719952901204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,128,0,1,fp8,fp8,0,1.89082670211792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,128,0,1,float16,fp8,0,2.3977227210998535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,128,0,1,float16,fp8,0,1.178213357925415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,128,0,1,fp8,fp8,0,0.9017706712086996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,128,0,1,fp8,fp8,0,0.9500906467437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,128,0,1,float16,float16,0,1.2389012972513835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,128,0,1,float16,fp8,0,1.20852263768514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,128,0,1,float16,float16,0,1.2098933060963948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,128,0,1,fp8,fp8,0,1.0012319882710774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,128,0,1,float16,fp8,0,1.1876906553904216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,128,0,1,float16,float16,0,0.7752640247344971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,128,0,1,float16,float16,0,1.2075413068135579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,128,0,1,float16,fp8,0,1.184330701828003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,128,0,1,fp8,fp8,0,0.6292586723963419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,128,0,1,fp8,fp8,0,1.0012426376342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,128,0,1,float16,fp8,0,0.6636106570561727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,128,0,1,float16,fp8,0,0.6323146820068359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,128,0,1,float16,float16,0,0.6110293467839559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,128,0,1,float16,fp8,0,0.6455413500467936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,128,0,1,fp8,fp8,0,0.487557331720988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,128,0,1,fp8,fp8,0,0.5257279872894287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,128,0,1,float16,float16,0,0.59661865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,128,0,1,float16,float16,0,0.5894506772359213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,128,0,1,float16,fp8,0,0.5898079872131348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,128,0,1,fp8,fp8,0,0.5030933221181234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,128,0,1,float16,float16,0,0.597653349240621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,128,0,1,fp8,fp8,0,0.30219733715057373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,128,0,1,float16,fp8,0,0.6512213150660197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,128,0,1,fp8,fp8,0,0.4949066638946533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,128,0,1,float16,float16,0,0.38094933827718097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,128,0,1,float16,fp8,0,0.3630186716715495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,128,0,1,float16,float16,0,0.3251306613286336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,128,0,1,float16,fp8,0,0.3365066846211751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,128,0,1,fp8,fp8,0,0.25493866205215454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,128,0,1,float16,float16,0,0.32468267281850177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,128,0,1,float16,fp8,0,0.33555201689402264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,128,0,1,fp8,fp8,0,0.2602399984995524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,128,0,1,float16,float16,0,0.3303893407185872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,128,0,1,float16,fp8,0,0.3359200159708659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,128,0,1,fp8,fp8,0,0.2585226694742839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,128,0,1,float16,float16,0,0.33243733644485474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,128,0,1,fp8,fp8,0,0.25804799795150757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,128,0,1,float16,fp8,0,0.33248533805211383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,128,0,1,float16,float16,0,0.19874133666356406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,128,0,1,float16,fp8,0,0.20268267393112183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,128,0,1,float16,float16,0,0.181167999903361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,128,0,1,fp8,fp8,0,0.1702773372332255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,128,0,1,float16,float16,0,0.18559465805689493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,128,0,1,float16,float16,0,0.18582399686177573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,128,0,1,float16,fp8,0,0.18420799573262533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,128,0,1,fp8,fp8,0,0.15588266650835672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,128,0,1,float16,fp8,0,0.18715733289718628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,128,0,1,fp8,fp8,0,0.15519466996192932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,128,0,1,float16,fp8,0,0.18247467279434204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,128,0,1,fp8,fp8,0,0.15532267093658447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,128,0,1,float16,float16,0,0.18644267320632935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,128,0,1,float16,fp8,0,0.18842132886250815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,128,0,1,fp8,fp8,0,0.1562933325767517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,128,0,1,fp8,fp8,0,4.763600031534831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,128,0,1,fp8,fp8,0,4.722069422403972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,128,0,1,float16,float16,0,6.098970413208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,128,0,1,float16,fp8,0,6.039040247599284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,128,0,1,float16,float16,0,6.037610371907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,128,0,1,float16,fp8,0,5.996010462443034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,128,0,1,float16,float16,0,6.131418863932292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,128,0,1,float16,fp8,0,5.9565175374348955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,128,0,1,fp8,fp8,0,4.746021270751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,128,0,1,float16,float16,0,3.414266586303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,128,0,1,fp8,fp8,0,2.7476107279459634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,128,0,1,float16,fp8,0,3.397024154663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,128,0,1,float16,float16,0,2.9700212478637695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,128,0,1,fp8,fp8,0,4.778794606526692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,128,0,1,float16,float16,0,6.070976257324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,128,0,1,float16,fp8,0,6.239797592163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,128,0,1,fp8,fp8,0,2.349829355875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,128,0,1,fp8,fp8,0,2.368271986643473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,128,0,1,float16,fp8,0,2.9075838724772134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,128,0,1,float16,float16,0,3.0371999740600586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,128,0,1,float16,fp8,0,2.9495251973470054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,128,0,1,float16,float16,0,2.899973233540853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,128,0,1,fp8,fp8,0,2.3767627080281577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,128,0,1,float16,fp8,0,3.045194625854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,128,0,1,float16,float16,0,1.689733346303304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,128,0,1,float16,float16,0,3.0176480611165366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,128,0,1,fp8,fp8,0,2.434703985850016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,128,0,1,float16,fp8,0,2.990901311238607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,128,0,1,float16,fp8,0,1.6763092676798503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,128,0,1,fp8,fp8,0,1.3894294102986653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,128,0,1,float16,float16,0,1.4331520398457844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,128,0,1,float16,fp8,0,1.431162675221761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,128,0,1,fp8,fp8,0,1.1958879629770915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,128,0,1,float16,float16,0,1.4119359652201335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,128,0,1,float16,fp8,0,1.394037405649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,128,0,1,fp8,fp8,0,1.2059306303660076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,128,0,1,float16,float16,0,1.3890719413757324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,128,0,1,float16,fp8,0,1.4592266082763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,128,0,1,fp8,fp8,0,1.2027520338694255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,128,0,1,float16,float16,0,1.4202027320861816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,128,0,1,float16,fp8,0,1.4687786102294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,128,0,1,fp8,fp8,0,1.1928213437398274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,128,0,1,float16,float16,0,0.8537226517995199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,128,0,1,float16,fp8,0,0.851685365041097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,128,0,1,fp8,fp8,0,0.731274684270223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,128,0,1,float16,float16,0,0.7119946479797363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,128,0,1,float16,fp8,0,0.7376746336619059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,128,0,1,fp8,fp8,0,0.6315253178278605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,128,0,1,float16,float16,0,0.7132053375244141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,128,0,1,fp8,fp8,0,0.6201546589533488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,128,0,1,float16,fp8,0,0.7268693447113037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,128,0,1,fp8,fp8,0,0.6229866743087769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,128,0,1,float16,float16,0,0.7325812975565592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,128,0,1,float16,fp8,0,0.7242133617401123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,128,0,1,float16,fp8,0,0.7490239938100179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,128,0,1,float16,float16,0,0.4471679925918579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,128,0,1,fp8,fp8,0,0.6303199927012125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,128,0,1,fp8,fp8,0,0.38355199495951336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,128,0,1,float16,fp8,0,0.44364798069000244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,128,0,1,float16,float16,0,0.7297333081563314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,128,0,1,float16,float16,0,0.3890933195749919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,128,0,1,float16,fp8,0,0.38446935017903644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,128,0,1,fp8,fp8,0,0.3396799961725871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,128,0,1,float16,float16,0,0.3904213507970174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,128,0,1,float16,fp8,0,0.392741322517395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,128,0,1,fp8,fp8,0,0.3352053165435791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,128,0,1,float16,float16,0,0.39217066764831543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,128,0,1,float16,float16,0,0.3887893358866374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,128,0,1,float16,fp8,0,0.389850656191508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,128,0,1,fp8,fp8,0,0.3390880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,128,0,1,float16,fp8,0,0.40134398142496747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,128,0,1,fp8,fp8,0,0.3423680067062378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,128,0,1,float16,float16,0,0.24385066827138266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,128,0,1,float16,fp8,0,0.24476800362269083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,128,0,1,fp8,fp8,0,0.2089866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,128,0,1,float16,float16,0,0.219760000705719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,128,0,1,float16,fp8,0,0.2206453283627828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,128,0,1,fp8,fp8,0,0.17921066284179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,128,0,1,float16,float16,0,0.22348799308141074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,128,0,1,float16,fp8,0,0.22430932521820068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,128,0,1,fp8,fp8,0,0.179530660311381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,128,0,1,float16,float16,0,0.22268799940745035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,128,0,1,float16,fp8,0,0.22477867205937704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,128,0,1,fp8,fp8,0,0.17891200383504233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,128,0,1,float16,float16,0,0.2230293353398641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,128,0,1,float16,fp8,0,0.22739199797312418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,128,0,1,fp8,fp8,0,0.18224533398946127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,128,0,1,float16,float16,0,0.13823466499646506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,128,0,1,float16,fp8,0,0.14114133516947427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,128,0,1,float16,float16,0,0.12743999560674033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,128,0,1,fp8,fp8,0,0.12329600254694621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,128,0,1,float16,float16,0,0.12662399808565775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,128,0,1,float16,fp8,0,0.1274720033009847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,128,0,1,fp8,fp8,0,0.10929600397745769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,128,0,1,float16,fp8,0,0.1290880044301351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,128,0,1,fp8,fp8,0,0.11170132954915364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,128,0,1,float16,float16,0,0.12771200140317282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,128,0,1,float16,fp8,0,0.12851732969284058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,128,0,1,fp8,fp8,0,0.11115733782450359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,128,0,1,float16,float16,0,0.1290720005830129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,128,0,1,float16,fp8,0,0.12946666280428568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,128,0,1,fp8,fp8,0,0.11248532931009929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,128,0,1,fp8,fp8,0,5.266565322875977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,128,0,1,float16,fp8,0,5.896938959757487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,128,0,1,float16,float16,0,6.075077056884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,128,0,1,fp8,fp8,0,5.187322616577148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,128,0,1,float16,float16,0,5.983397165934245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,128,0,1,float16,fp8,0,6.029962539672852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,128,0,1,float16,float16,0,6.373061498006185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,128,0,1,float16,fp8,0,6.292191823323567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,128,0,1,float16,float16,0,2.8766345977783203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,128,0,1,float16,float16,0,3.8647146224975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,128,0,1,fp8,fp8,0,3.1944265365600586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,128,0,1,float16,fp8,0,3.8252693812052407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,128,0,1,fp8,fp8,0,5.580058415730794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,128,0,1,fp8,fp8,0,5.615578969319661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,128,0,1,float16,float16,0,6.511610666910808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,128,0,1,float16,fp8,0,6.379434585571289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,128,0,1,fp8,fp8,0,2.666266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,128,0,1,float16,fp8,0,2.9511680603027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,128,0,1,float16,float16,0,2.952154795328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,128,0,1,float16,fp8,0,3.0181652704874673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,128,0,1,fp8,fp8,0,2.5859840710957847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,128,0,1,float16,float16,0,3.1704107920328775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,128,0,1,float16,fp8,0,2.9736852645874023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,128,0,1,float16,float16,0,1.9364800453186035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,128,0,1,fp8,fp8,0,2.869632085164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,128,0,1,float16,float16,0,3.096965471903483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,128,0,1,float16,fp8,0,1.8669546445210774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,128,0,1,fp8,fp8,0,2.8372694651285806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,128,0,1,float16,fp8,0,3.0388800303141275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,128,0,1,fp8,fp8,0,1.605456034342448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,128,0,1,float16,float16,0,1.4655253092447917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,128,0,1,fp8,fp8,0,1.3043999671936035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,128,0,1,float16,fp8,0,1.469744046529134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,128,0,1,float16,float16,0,1.4701333045959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,128,0,1,float16,fp8,0,1.4557387034098308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,128,0,1,fp8,fp8,0,1.341861406962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,128,0,1,float16,float16,0,1.4968852996826172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,128,0,1,fp8,fp8,0,1.403674602508545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,128,0,1,float16,fp8,0,1.4827040036519368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,128,0,1,float16,float16,0,1.5118826230367024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,128,0,1,float16,float16,0,0.9754400253295898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,128,0,1,float16,fp8,0,0.9375627040863037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,128,0,1,float16,fp8,0,1.501962661743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,128,0,1,fp8,fp8,0,1.4540425936381023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,128,0,1,fp8,fp8,0,0.8083733717600504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,128,0,1,float16,float16,0,0.7562719980875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,128,0,1,float16,fp8,0,0.7500747044881185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,128,0,1,fp8,fp8,0,0.6619040171305338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,128,0,1,float16,float16,0,0.75054399172465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,128,0,1,float16,fp8,0,0.7661866346995035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,128,0,1,fp8,fp8,0,0.665669322013855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,128,0,1,float16,float16,0,0.7607200145721436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,128,0,1,fp8,fp8,0,0.7340906461079916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,128,0,1,float16,fp8,0,0.7565279801686605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,128,0,1,float16,float16,0,0.76091202100118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,128,0,1,float16,float16,0,0.4973599910736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,128,0,1,fp8,fp8,0,0.7421759764353434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,128,0,1,float16,fp8,0,0.7674132982889811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,128,0,1,float16,fp8,0,0.4792160193125407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,128,0,1,fp8,fp8,0,0.4154026508331299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,128,0,1,float16,fp8,0,0.3875519831975301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,128,0,1,fp8,fp8,0,0.33882665634155273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,128,0,1,float16,float16,0,0.3896160125732422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,128,0,1,float16,float16,0,0.3882720073064168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,128,0,1,float16,fp8,0,0.38653333981831867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,128,0,1,fp8,fp8,0,0.3405386606852214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,128,0,1,float16,float16,0,0.39289601643880206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,128,0,1,float16,fp8,0,0.3935893376668294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,128,0,1,fp8,fp8,0,0.372437318166097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,128,0,1,float16,float16,0,0.39449600378672284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,128,0,1,float16,fp8,0,0.3957706689834595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,128,0,1,float16,float16,0,0.2619946599006653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,128,0,1,fp8,fp8,0,0.3745439847310384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,128,0,1,float16,fp8,0,0.2545439998308818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,128,0,1,fp8,fp8,0,0.21659199396769205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,128,0,1,float16,float16,0,0.20800000429153442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,128,0,1,float16,fp8,0,0.20785599946975708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,128,0,1,fp8,fp8,0,0.16126933693885803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,128,0,1,float16,float16,0,0.20721065998077393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,128,0,1,float16,fp8,0,0.20580265919367471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,128,0,1,fp8,fp8,0,0.16503999630610147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,128,0,1,float16,float16,0,0.21056532859802246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,128,0,1,float16,fp8,0,0.2097813288370768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,128,0,1,fp8,fp8,0,0.1760586698849996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,128,0,1,float16,fp8,0,0.14739200472831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,128,0,1,fp8,fp8,0,0.11242133378982544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,128,0,1,float16,float16,0,0.21368533372879028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,128,0,1,float16,fp8,0,0.20992533365885416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,128,0,1,float16,float16,0,0.11034666498502095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,128,0,1,fp8,fp8,0,0.1771999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,128,0,1,float16,float16,0,0.15103999773661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,128,0,1,float16,float16,0,0.1092639962832133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,128,0,1,float16,fp8,0,0.11029866337776184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,128,0,1,fp8,fp8,0,0.09399466713269551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,128,0,1,fp8,fp8,0,0.088837335507075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,128,0,1,float16,fp8,0,0.10993599891662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,128,0,1,fp8,fp8,0,0.08854933579762776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,128,0,1,float16,float16,0,0.11266666650772095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,128,0,1,float16,fp8,0,0.11065066854159038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,128,0,1,float16,float16,0,0.11255466938018799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,128,0,1,float16,fp8,0,0.11376532912254333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,128,0,1,fp8,fp8,0,0.09579199552536011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,128,0,1,float16,float16,0,0.07320533196131389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,128,0,1,float16,fp8,0,0.07133333384990692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,128,0,1,fp8,fp8,0,0.06470400094985962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,128,0,1,float16,float16,0,0.05945600072542826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,128,0,1,float16,fp8,0,0.05869866907596588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,128,0,1,fp8,fp8,0,0.049728001157442726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,128,0,1,float16,float16,0,0.05996799965699514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,128,0,1,float16,fp8,0,0.059893334905306496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,128,0,1,fp8,fp8,0,0.05053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,128,0,1,float16,float16,0,0.061018665631612144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,128,0,1,float16,fp8,0,0.0610453337430954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,128,0,1,fp8,fp8,0,0.053743998209635414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,128,0,1,float16,float16,0,0.06085866689682007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,128,0,1,float16,fp8,0,0.06257600088914235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,128,0,1,fp8,fp8,0,0.0551146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,128,0,1,fp8,fp8,0,4.235653241475423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,128,0,1,float16,float16,0,4.521802584330241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,128,0,1,float16,fp8,0,4.511861483256022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,128,0,1,float16,float16,0,4.491962750752767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,128,0,1,fp8,fp8,0,4.258101463317871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,128,0,1,float16,fp8,0,4.558133443196614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,128,0,1,float16,float16,0,4.87666130065918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,128,0,1,float16,float16,0,3.2193867365519204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,128,0,1,float16,fp8,0,4.763279914855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,128,0,1,fp8,fp8,0,4.639194806416829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,128,0,1,float16,fp8,0,3.083258628845215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,128,0,1,float16,float16,0,5.016895929972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,128,0,1,float16,fp8,0,4.617557207743327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,128,0,1,fp8,fp8,0,4.687562624613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,128,0,1,float16,float16,0,2.2908479372660318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,128,0,1,fp8,fp8,0,2.7084267934163413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,128,0,1,float16,fp8,0,2.2507413228352866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,128,0,1,fp8,fp8,0,2.1540212631225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,128,0,1,float16,float16,0,2.284053325653076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,128,0,1,float16,fp8,0,2.2666346232096353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,128,0,1,fp8,fp8,0,2.1133599281311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,128,0,1,float16,float16,0,2.4679147402445474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,128,0,1,float16,fp8,0,2.2767945925394693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,128,0,1,fp8,fp8,0,2.3098079363505044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,128,0,1,float16,float16,0,1.6100640296936035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,128,0,1,fp8,fp8,0,1.357375939687093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,128,0,1,float16,fp8,0,1.5418507258097331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,128,0,1,float16,fp8,0,2.3470667203267417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,128,0,1,float16,float16,0,2.3337225914001465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,128,0,1,fp8,fp8,0,2.3638933499654136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,128,0,1,float16,float16,0,1.1396533648173015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,128,0,1,fp8,fp8,0,1.0810826619466145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,128,0,1,float16,fp8,0,1.1393760045369465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,128,0,1,float16,float16,0,1.1573599974314372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,128,0,1,float16,fp8,0,1.1340373357137044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,128,0,1,fp8,fp8,0,1.0585546493530273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,128,0,1,float16,float16,0,1.1502026716868083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,128,0,1,float16,fp8,0,1.1562506357828777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,128,0,1,fp8,fp8,0,1.1650666395823162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,128,0,1,float16,float16,0,0.8062240282694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,128,0,1,float16,fp8,0,0.7769227027893066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,128,0,1,float16,float16,0,1.1895946661631267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,128,0,1,float16,fp8,0,1.181109348932902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,128,0,1,fp8,fp8,0,1.2062880198160808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,128,0,1,fp8,fp8,0,0.6684479713439941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,128,0,1,float16,float16,0,0.5847146511077881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,128,0,1,float16,fp8,0,0.5885813236236572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,128,0,1,fp8,fp8,0,0.5216586589813232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,128,0,1,float16,float16,0,0.580677350362142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,128,0,1,float16,fp8,0,0.5874240001042684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,128,0,1,fp8,fp8,0,0.5359786748886108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,128,0,1,float16,float16,0,0.6000106732050577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,128,0,1,float16,fp8,0,0.5819093386332194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,128,0,1,fp8,fp8,0,0.5986080169677734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,128,0,1,float16,float16,0,0.6064000129699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,128,0,1,float16,fp8,0,0.5925973256429037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,128,0,1,float16,float16,0,0.41791999340057373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,128,0,1,float16,fp8,0,0.39866665999094647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,128,0,1,fp8,fp8,0,0.6097120046615601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,128,0,1,float16,float16,0,0.2983786662419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,128,0,1,fp8,fp8,0,0.3468693494796753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,128,0,1,float16,fp8,0,0.30269867181777954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,128,0,1,fp8,fp8,0,0.2754240036010742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,128,0,1,float16,float16,0,0.2997973362604777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,128,0,1,float16,fp8,0,0.30238399902979535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,128,0,1,fp8,fp8,0,0.27569599946339923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,128,0,1,float16,float16,0,0.3097280065218608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,128,0,1,float16,fp8,0,0.30642133951187134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,128,0,1,fp8,fp8,0,0.30670400460561115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,128,0,1,float16,float16,0,0.3129333257675171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,128,0,1,float16,fp8,0,0.31302400430043537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,128,0,1,fp8,fp8,0,0.3017653425534566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,128,0,1,float16,float16,0,0.2185653249422709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,128,0,1,float16,fp8,0,0.21075199047724405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,128,0,1,fp8,fp8,0,0.18573333819707236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,128,0,1,float16,float16,0,0.1622933348019918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,128,0,1,float16,fp8,0,0.16317866245905557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,128,0,1,fp8,fp8,0,0.13358400265375772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,128,0,1,float16,float16,0,0.16361600160598755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,128,0,1,float16,fp8,0,0.16461867094039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,128,0,1,float16,fp8,0,0.1620266636212667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,128,0,1,fp8,fp8,0,0.1360373298327128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,128,0,1,float16,float16,0,0.16447466611862183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,128,0,1,fp8,fp8,0,0.14829867084821066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,128,0,1,float16,float16,0,0.1692053278287252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,128,0,1,float16,fp8,0,0.16743467251459757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,128,0,1,fp8,fp8,0,0.15145066380500793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,128,0,1,float16,float16,0,0.13176000118255615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,128,0,1,float16,fp8,0,0.12853866815567017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,128,0,1,fp8,fp8,0,0.09709333380063374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,128,0,1,float16,float16,0,0.0881226658821106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,128,0,1,float16,fp8,0,0.08737066388130188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,128,0,1,float16,fp8,0,0.09051733215649922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,128,0,1,fp8,fp8,0,0.07381866872310638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,128,0,1,float16,float16,0,0.08833600083986919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,128,0,1,float16,fp8,0,0.08905067046483357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,128,0,1,fp8,fp8,0,0.07514133552710216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,128,0,1,float16,float16,0,0.08940800031026204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,128,0,1,fp8,fp8,0,0.0782239983479182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,128,0,1,fp8,fp8,0,0.057205334305763245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,128,0,1,float16,float16,0,0.09093333284060161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,128,0,1,float16,fp8,0,0.09149332841237386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,128,0,1,fp8,fp8,0,0.0812799980243047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,128,0,1,float16,float16,0,0.06413866579532623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,128,0,1,float16,fp8,0,0.062368000547091164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,128,0,1,float16,float16,0,0.04980800052483877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,128,0,1,float16,float16,0,0.04955733319123586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,128,0,1,float16,fp8,0,0.05045866469542185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,128,0,1,fp8,fp8,0,0.042992000778516136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,128,0,1,float16,fp8,0,0.0509493350982666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,128,0,1,fp8,fp8,0,0.043178667624791466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,128,0,1,float16,float16,0,0.05089066425959269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,128,0,1,float16,fp8,0,0.051072001457214355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,128,0,1,fp8,fp8,0,0.04675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,128,0,1,fp8,fp8,0,0.046165332198143005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,128,0,1,float16,float16,0,0.052629331747690834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,128,0,1,float16,fp8,0,0.05171733101209005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,128,0,1,float16,float16,0,0.036389333506425224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,128,0,1,float16,fp8,0,0.035216001172860466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,128,0,1,fp8,fp8,0,0.033189333975315094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,128,0,1,float16,float16,0,0.031888000667095184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,128,0,1,float16,fp8,0,0.03141333411137263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,128,0,1,fp8,fp8,0,0.029205332199732464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,128,0,1,float16,float16,0,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,128,0,1,float16,fp8,0,0.03242133309443792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,128,0,1,fp8,fp8,0,0.02938666691382726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,128,0,1,float16,float16,0,0.03230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,128,0,1,float16,fp8,0,0.03281066566705704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,128,0,1,fp8,fp8,0,0.030410667260487873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,128,0,1,float16,float16,0,0.03286933402220408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,128,0,1,float16,fp8,0,0.03259200106064478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,128,0,1,fp8,fp8,0,0.03089066594839096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,128,0,1,fp8,fp8,0,1.7334507306416829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,128,0,1,float16,float16,0,1.949600060780843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,128,0,1,float16,fp8,0,1.9520907402038574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,128,0,1,fp8,fp8,0,1.7261706988016765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,128,0,1,float16,float16,0,1.962565263112386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,128,0,1,float16,fp8,0,1.9456000328063965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,128,0,1,float16,float16,0,2.035360018412272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,128,0,1,float16,float16,0,1.479098637898763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,128,0,1,float16,fp8,0,1.991125265757243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,128,0,1,fp8,fp8,0,1.9305226008097331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,128,0,1,float16,float16,0,2.017594655354818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,128,0,1,float16,fp8,0,1.4130187034606934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,128,0,1,float16,fp8,0,2.0046399434407554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,128,0,1,fp8,fp8,0,1.9857920010884602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,128,0,1,fp8,fp8,0,1.1570613384246826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,128,0,1,float16,float16,0,0.9836053053538004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,128,0,1,float16,fp8,0,0.9846773147583008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,128,0,1,fp8,fp8,0,0.8706346352895101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,128,0,1,float16,float16,0,0.9843573570251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,128,0,1,fp8,fp8,0,0.8462826410929362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,128,0,1,float16,fp8,0,0.9891946315765381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,128,0,1,float16,float16,0,1.0204906463623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,128,0,1,fp8,fp8,0,0.9762720266977946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,128,0,1,float16,fp8,0,0.9996426900227865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,128,0,1,float16,float16,0,0.7422239780426025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,128,0,1,float16,fp8,0,0.7173813184102377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,128,0,1,float16,float16,0,1.0286880334218342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,128,0,1,float16,fp8,0,1.0209120114644368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,128,0,1,fp8,fp8,0,0.9988853136698405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,128,0,1,fp8,fp8,0,0.5808746814727783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,128,0,1,float16,fp8,0,0.5032906532287598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,128,0,1,float16,float16,0,0.5053653319676717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,128,0,1,fp8,fp8,0,0.4222506682078044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,128,0,1,float16,float16,0,0.5071413516998291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,128,0,1,float16,fp8,0,0.5034720102945963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,128,0,1,fp8,fp8,0,0.4366399844487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,128,0,1,float16,float16,0,0.5151840051015218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,128,0,1,fp8,fp8,0,0.4906933307647705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,128,0,1,float16,fp8,0,0.5117706855138143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,128,0,1,float16,float16,0,0.5247146685918173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,128,0,1,float16,fp8,0,0.5142293373743693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,128,0,1,fp8,fp8,0,0.5057493448257446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,128,0,1,float16,float16,0,0.382368008295695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,128,0,1,float16,fp8,0,0.3653973340988159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,128,0,1,fp8,fp8,0,0.3025226593017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,128,0,1,float16,fp8,0,0.26266666253407794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,128,0,1,float16,float16,0,0.2600053350130717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,128,0,1,fp8,fp8,0,0.22377065817515054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,128,0,1,float16,float16,0,0.2611573338508606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,128,0,1,float16,fp8,0,0.26258667310078937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,128,0,1,fp8,fp8,0,0.22604266802469888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,128,0,1,float16,float16,0,0.2687679926554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,128,0,1,float16,fp8,0,0.2658613324165344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,128,0,1,fp8,fp8,0,0.2550133268038432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,128,0,1,float16,float16,0,0.270197331905365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,128,0,1,float16,fp8,0,0.27059199412663776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,128,0,1,fp8,fp8,0,0.2588106592496236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,128,0,1,float16,float16,0,0.20364266633987427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,128,0,1,float16,fp8,0,0.1952000061670939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,128,0,1,fp8,fp8,0,0.16703466574350992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,128,0,1,float16,float16,0,0.14194666345914206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,128,0,1,float16,fp8,0,0.14139200250307718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,128,0,1,fp8,fp8,0,0.11777599652608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,128,0,1,float16,float16,0,0.1414346694946289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,128,0,1,float16,fp8,0,0.14079999923706055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,128,0,1,fp8,fp8,0,0.12131733695665996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,128,0,1,float16,fp8,0,0.14459199706713358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,128,0,1,float16,float16,0,0.1450933317343394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,128,0,1,float16,fp8,0,0.1442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,128,0,1,fp8,fp8,0,0.13456533352533975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,128,0,1,float16,float16,0,0.1476640005906423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,128,0,1,fp8,fp8,0,0.1362879971663157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,128,0,1,float16,float16,0,0.12087999780972798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,128,0,1,float16,fp8,0,0.11687466502189636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,128,0,1,fp8,fp8,0,0.0906826655069987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,128,0,1,float16,float16,0,0.0784693310658137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,128,0,1,float16,fp8,0,0.07898666461308797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,128,0,1,fp8,fp8,0,0.06673599779605865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,128,0,1,fp8,fp8,0,0.0664106657107671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,128,0,1,float16,float16,0,0.07871466875076294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,128,0,1,float16,fp8,0,0.07826133569081624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,128,0,1,float16,fp8,0,0.08295999964078267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,128,0,1,float16,float16,0,0.08155733346939087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,128,0,1,float16,fp8,0,0.08212799827257793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,128,0,1,fp8,fp8,0,0.07172800103823344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,128,0,1,fp8,fp8,0,0.053226664662361145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,128,0,1,float16,float16,0,0.08340799808502197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,128,0,1,fp8,fp8,0,0.07326933244864146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,128,0,1,float16,float16,0,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,128,0,1,float16,fp8,0,0.05772800246874491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,128,0,1,float16,float16,0,0.04436799883842468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,128,0,1,float16,fp8,0,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,128,0,1,fp8,fp8,0,0.03792533278465271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,128,0,1,float16,float16,0,0.04557866851488749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,128,0,1,float16,fp8,0,0.04496533175309499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,128,0,1,fp8,fp8,0,0.03894400099913279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,128,0,1,float16,float16,0,0.046757335464159645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,128,0,1,float16,fp8,0,0.04606399933497111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,128,0,1,fp8,fp8,0,0.041002665956815086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,128,0,1,float16,float16,0,0.047093331813812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,128,0,1,float16,fp8,0,0.04725866516431173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,128,0,1,fp8,fp8,0,0.042805333932240806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,128,0,1,float16,fp8,0,0.0305173322558403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,128,0,1,float16,float16,0,0.0330826664964358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,128,0,1,float16,float16,0,0.03033600002527237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,128,0,1,float16,fp8,0,0.032618666688601174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,128,0,1,fp8,fp8,0,0.03149333347876867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,128,0,1,float16,float16,0,0.03019733230272929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,128,0,1,fp8,fp8,0,0.026975999275843304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,128,0,1,float16,fp8,0,0.03081600119670232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,128,0,1,fp8,fp8,0,0.02720533311367035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,128,0,1,float16,float16,0,0.031093334158261616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,128,0,1,float16,fp8,0,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,128,0,1,fp8,fp8,0,0.028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,128,0,1,float16,float16,0,0.031002665559450786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,128,0,1,float16,fp8,0,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,128,0,1,fp8,fp8,0,0.028794666131337483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,128,0,1,float16,float16,0,0.023930666347344715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,128,0,1,float16,fp8,0,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,128,0,1,fp8,fp8,0,0.022453332940737408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,128,0,1,float16,float16,0,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,128,0,1,float16,fp8,0,0.022869333624839783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,128,0,1,fp8,fp8,0,0.020080000162124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,128,0,1,float16,float16,0,0.021840001145998638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,128,0,1,float16,fp8,0,0.022474666436513264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,128,0,1,fp8,fp8,0,0.02014933278163274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,128,0,1,float16,float16,0,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,128,0,1,fp8,fp8,0,0.021536000072956085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,128,0,1,float16,fp8,0,0.023738667368888855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,128,0,1,fp8,fp8,0,0.021546666820844013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,128,0,1,float16,float16,0,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,128,0,1,float16,fp8,0,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,128,0,1,float16,float16,0,0.9012693564097086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,128,0,1,fp8,fp8,0,0.8533279895782471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,128,0,1,float16,fp8,0,0.9095040162404379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,128,0,1,float16,float16,0,0.9197973410288492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,128,0,1,float16,fp8,0,0.9127146402994791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,128,0,1,fp8,fp8,0,0.8737653096516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,128,0,1,float16,float16,0,0.9888532956441244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,128,0,1,float16,fp8,0,0.9411306381225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,128,0,1,fp8,fp8,0,0.9635999997456869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,128,0,1,float16,float16,0,0.9590880076090494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,128,0,1,float16,float16,0,0.7658027013142904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,128,0,1,float16,fp8,0,0.9505333105723063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,128,0,1,fp8,fp8,0,0.5870346625645956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,128,0,1,fp8,fp8,0,0.990501324335734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,128,0,1,float16,fp8,0,0.7308586438496908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,128,0,1,float16,float16,0,0.4638506571451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,128,0,1,float16,fp8,0,0.4628959894180298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,128,0,1,fp8,fp8,0,0.4215466578801473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,128,0,1,float16,float16,0,0.46880535284678143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,128,0,1,float16,fp8,0,0.46513601144154865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,128,0,1,fp8,fp8,0,0.4354933500289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,128,0,1,float16,float16,0,0.4846506516138713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,128,0,1,float16,fp8,0,0.4795999924341838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,128,0,1,fp8,fp8,0,0.4900746742884318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,128,0,1,float16,float16,0,0.4934613307317098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,128,0,1,float16,fp8,0,0.48924799760182697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,128,0,1,fp8,fp8,0,0.49990399678548175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,128,0,1,float16,float16,0,0.38955732186635333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,128,0,1,float16,fp8,0,0.37993065516153973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,128,0,1,float16,float16,0,0.24310400088628134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,128,0,1,fp8,fp8,0,0.29734400908152264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,128,0,1,float16,fp8,0,0.24316267172495523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,128,0,1,fp8,fp8,0,0.21934932470321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,128,0,1,float16,float16,0,0.2460426688194275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,128,0,1,float16,fp8,0,0.24493332703908285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,128,0,1,fp8,fp8,0,0.22261865933736166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,128,0,1,float16,fp8,0,0.25115732351938885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,128,0,1,float16,float16,0,0.2563839952150981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,128,0,1,fp8,fp8,0,0.2513279914855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,128,0,1,float16,float16,0,0.26013867060343426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,128,0,1,float16,fp8,0,0.2560693422953288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,128,0,1,fp8,fp8,0,0.25998934110005695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,128,0,1,float16,float16,0,0.2070293426513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,128,0,1,float16,fp8,0,0.1993066668510437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,128,0,1,fp8,fp8,0,0.157669335603714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,128,0,1,float16,float16,0,0.13135466972986856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,128,0,1,float16,fp8,0,0.13193600376447043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,128,0,1,fp8,fp8,0,0.11923199892044067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,128,0,1,float16,float16,0,0.13409066200256348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,128,0,1,float16,fp8,0,0.13239999612172446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,128,0,1,fp8,fp8,0,0.12117866675059001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,128,0,1,float16,float16,0,0.13969600200653076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,128,0,1,float16,fp8,0,0.14114133516947427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,128,0,1,float16,fp8,0,0.1374506652355194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,128,0,1,fp8,fp8,0,0.132533331712087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,128,0,1,float16,float16,0,0.14268799622853598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,128,0,1,fp8,fp8,0,0.13362133502960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,128,0,1,float16,float16,0,0.11291733384132385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,128,0,1,float16,fp8,0,0.10946666200955708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,128,0,1,fp8,fp8,0,0.08800533413887024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,128,0,1,float16,float16,0,0.07246399919191997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,128,0,1,float16,fp8,0,0.07382399837176006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,128,0,1,fp8,fp8,0,0.06533333162466685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,128,0,1,float16,float16,0,0.07343466579914093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,128,0,1,float16,fp8,0,0.07379733522733052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,128,0,1,fp8,fp8,0,0.06545066833496094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,128,0,1,float16,float16,0,0.07655466596285503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,128,0,1,float16,fp8,0,0.07578133543332417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,128,0,1,fp8,fp8,0,0.07036800185839336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,128,0,1,float16,float16,0,0.07898666461308797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,128,0,1,float16,fp8,0,0.07830933233102162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,128,0,1,fp8,fp8,0,0.07309333483378093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,128,0,1,float16,float16,0,0.05961066484451294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,128,0,1,float16,fp8,0,0.05736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,128,0,1,fp8,fp8,0,0.051445335149765015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,128,0,1,float16,float16,0,0.043178667624791466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,128,0,1,float16,fp8,0,0.042992000778516136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,128,0,1,fp8,fp8,0,0.037808001041412354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,128,0,1,float16,float16,0,0.0430026650428772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,128,0,1,float16,fp8,0,0.04398400088151296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,128,0,1,fp8,fp8,0,0.037946666280428566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,128,0,1,float16,float16,0,0.04452266792456309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,128,0,1,float16,fp8,0,0.04462933540344238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,128,0,1,fp8,fp8,0,0.04234666625658671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,128,0,1,float16,fp8,0,0.03297599901755651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,128,0,1,float16,float16,0,0.04533333579699198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,128,0,1,float16,fp8,0,0.04544533292452494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,128,0,1,fp8,fp8,0,0.042463997999827065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,128,0,1,float16,float16,0,0.03335466732581457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,128,0,1,fp8,fp8,0,0.030821333328882854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,128,0,1,float16,float16,0,0.029834667841593426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,128,0,1,float16,fp8,0,0.030159999926884968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,128,0,1,fp8,fp8,0,0.026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,128,0,1,float16,float16,0,0.02998399982849757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,128,0,1,float16,fp8,0,0.0308693324526151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,128,0,1,fp8,fp8,0,0.027114666998386383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,128,0,1,float16,float16,0,0.030949334303538006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,128,0,1,float16,fp8,0,0.031498665610949196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,128,0,1,fp8,fp8,0,0.028442665934562683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,128,0,1,float16,float16,0,0.030613332986831665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,128,0,1,float16,float16,0,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,128,0,1,float16,fp8,0,0.031354665756225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,128,0,1,fp8,fp8,0,0.028394666810830433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,128,0,1,float16,float16,0,0.02277333289384842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,128,0,1,float16,fp8,0,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,128,0,1,float16,fp8,0,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,128,0,1,fp8,fp8,0,0.022319999833901722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,128,0,1,float16,fp8,0,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,128,0,1,fp8,fp8,0,0.019989332805077236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,128,0,1,float16,float16,0,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,128,0,1,fp8,fp8,0,0.020010666300853092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,128,0,1,float16,float16,0,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,128,0,1,float16,fp8,0,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,128,0,1,fp8,fp8,0,0.021664001047611237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,128,0,1,float16,float16,0,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,128,0,1,float16,fp8,0,0.022122666239738464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,128,0,1,fp8,fp8,0,0.020874666670958202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,128,0,1,float16,float16,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,128,0,1,float16,fp8,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,128,0,1,fp8,fp8,0,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,128,0,1,float16,float16,0,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,128,0,1,float16,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,128,0,1,fp8,fp8,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,128,0,1,float16,float16,0,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,128,0,1,float16,fp8,0,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,128,0,1,fp8,fp8,0,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,128,0,1,float16,float16,0,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,128,0,1,float16,fp8,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,128,0,1,fp8,fp8,0,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,128,0,1,float16,float16,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,128,0,1,float16,fp8,0,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,128,0,1,fp8,fp8,0,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,128,0,1,float16,float16,0,0.5313226779301962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,128,0,1,float16,fp8,0,0.5336800018946329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,128,0,1,fp8,fp8,0,0.5816160043080648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,128,0,1,float16,float16,0,0.5343413352966309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,128,0,1,float16,float16,0,0.5518506765365601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,128,0,1,float16,fp8,0,0.5341173410415649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,128,0,1,float16,fp8,0,0.5461440086364746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,128,0,1,fp8,fp8,0,0.597327987353007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,128,0,1,fp8,fp8,0,0.6561760107676188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,128,0,1,float16,float16,0,0.5575786828994751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,128,0,1,float16,float16,0,0.3929440180460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,128,0,1,float16,fp8,0,0.5493760108947754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,128,0,1,float16,fp8,0,0.3788906733194987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,128,0,1,fp8,fp8,0,0.37625598907470703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,128,0,1,float16,float16,0,0.2733280062675476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,128,0,1,fp8,fp8,0,0.6671733061472574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,128,0,1,float16,fp8,0,0.2735840082168579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,128,0,1,fp8,fp8,0,0.3013333280881246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,128,0,1,float16,float16,0,0.27380800247192383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,128,0,1,float16,fp8,0,0.2768746614456177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,128,0,1,fp8,fp8,0,0.30402666330337524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,128,0,1,float16,float16,0,0.28172266483306885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,128,0,1,float16,fp8,0,0.279530664285024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,128,0,1,fp8,fp8,0,0.33156800270080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,128,0,1,float16,float16,0,0.2882293264071147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,128,0,1,float16,fp8,0,0.28562132517496747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,128,0,1,float16,float16,0,0.20621333519617716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,128,0,1,fp8,fp8,0,0.3412586847941081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,128,0,1,float16,fp8,0,0.199018657207489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,128,0,1,fp8,fp8,0,0.19669334093729654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,128,0,1,float16,float16,0,0.14560533563296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,128,0,1,float16,fp8,0,0.14575466513633728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,128,0,1,fp8,fp8,0,0.1588746706644694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,128,0,1,float16,float16,0,0.14497599999109903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,128,0,1,float16,fp8,0,0.1451359987258911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,128,0,1,fp8,fp8,0,0.160970667997996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,128,0,1,float16,float16,0,0.14949867129325867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,128,0,1,float16,fp8,0,0.15007999539375305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,128,0,1,fp8,fp8,0,0.1715573271115621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,128,0,1,float16,float16,0,0.15262933572133383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,128,0,1,float16,fp8,0,0.1520906686782837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,128,0,1,fp8,fp8,0,0.17798399925231934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,128,0,1,float16,float16,0,0.1090773344039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,128,0,1,float16,fp8,0,0.10522133111953735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,128,0,1,float16,float16,0,0.07699733475844066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,128,0,1,fp8,fp8,0,0.10803199807802837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,128,0,1,float16,fp8,0,0.07711466650168101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,128,0,1,fp8,fp8,0,0.08662933111190796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,128,0,1,float16,fp8,0,0.08032000064849854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,128,0,1,float16,float16,0,0.0786186655362447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,128,0,1,float16,fp8,0,0.07751999795436859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,128,0,1,fp8,fp8,0,0.0874079962571462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,128,0,1,float16,float16,0,0.07960000137488048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,128,0,1,fp8,fp8,0,0.09174399574597676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,128,0,1,float16,float16,0,0.08167999982833862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,128,0,1,float16,fp8,0,0.08269333342711131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,128,0,1,fp8,fp8,0,0.0956160028775533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,128,0,1,float16,float16,0,0.060085331400235496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,128,0,1,float16,fp8,0,0.05863999823729197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,128,0,1,fp8,fp8,0,0.06226666768391927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,128,0,1,float16,float16,0,0.0444106658299764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,128,0,1,float16,fp8,0,0.04483733574549357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,128,0,1,fp8,fp8,0,0.04910400013128916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,128,0,1,float16,fp8,0,0.04729066789150238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,128,0,1,float16,float16,0,0.04525866607824961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,128,0,1,float16,fp8,0,0.04533333579699198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,128,0,1,fp8,fp8,0,0.04962133367856344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,128,0,1,float16,float16,0,0.04683200021584829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,128,0,1,fp8,fp8,0,0.052613332867622375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,128,0,1,float16,float16,0,0.04762133459250132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,128,0,1,float16,fp8,0,0.048751999934514366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,128,0,1,fp8,fp8,0,0.0537066658337911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,128,0,1,float16,fp8,0,0.02887466549873352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,128,0,1,float16,float16,0,0.03209600100914637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,128,0,1,float16,fp8,0,0.031157332162062328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,128,0,1,fp8,fp8,0,0.035749333600203194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,128,0,1,float16,float16,0,0.028783999383449554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,128,0,1,fp8,fp8,0,0.030591999491055805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,128,0,1,float16,float16,0,0.02921066681543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,128,0,1,float16,fp8,0,0.028757333755493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,128,0,1,fp8,fp8,0,0.03091199944416682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,128,0,1,float16,float16,0,0.029285334050655365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,128,0,1,float16,fp8,0,0.029658667743206024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,128,0,1,fp8,fp8,0,0.03239466746648153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,128,0,1,float16,float16,0,0.029845332105954487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,128,0,1,float16,fp8,0,0.030058667063713074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,128,0,1,fp8,fp8,0,0.03272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,128,0,1,float16,float16,0,0.02437866727511088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,128,0,1,float16,fp8,0,0.02456533412138621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,128,0,1,fp8,fp8,0,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,128,0,1,float16,float16,0,0.02252800017595291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,128,0,1,float16,float16,0,0.023007998863856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,128,0,1,float16,fp8,0,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,128,0,1,fp8,fp8,0,0.023754666248957317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,128,0,1,float16,float16,0,0.02293866624434789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,128,0,1,float16,fp8,0,0.023061332603295643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,128,0,1,fp8,fp8,0,0.02405333270629247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,128,0,1,float16,fp8,0,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,128,0,1,fp8,fp8,0,0.024432001014550526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,128,0,1,float16,float16,0,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,128,0,1,float16,fp8,0,0.02330133318901062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,128,0,1,fp8,fp8,0,0.024383999407291412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,128,0,1,float16,float16,0,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,128,0,1,float16,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,128,0,1,float16,fp8,0,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,128,0,1,fp8,fp8,0,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,128,0,1,float16,float16,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,128,0,1,float16,fp8,0,0.016789333273967106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,128,0,1,fp8,fp8,0,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,128,0,1,float16,float16,0,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,128,0,1,fp8,fp8,0,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,128,0,1,float16,float16,0,0.016613333175579708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,128,0,1,float16,fp8,0,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,128,0,1,fp8,fp8,0,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,128,0,1,float16,float16,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,128,0,1,float16,fp8,0,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,128,0,1,fp8,fp8,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,128,0,1,float16,float16,0,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,128,0,1,float16,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,128,0,1,fp8,fp8,0,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,128,0,1,float16,float16,0,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,128,0,1,float16,fp8,0,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,128,0,1,fp8,fp8,0,0.016800000021855038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,128,0,1,float16,float16,0,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,128,0,1,float16,fp8,0,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,128,0,1,float16,fp8,0,0.016016000260909397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,128,0,1,fp8,fp8,0,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,128,0,1,float16,float16,0,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,128,0,1,float16,fp8,0,0.01584533353646596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,128,0,1,float16,float16,0,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,128,0,1,fp8,fp8,0,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,128,0,1,fp8,fp8,0,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,128,0,1,float16,float16,0,0.36368000507354736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,128,0,1,float16,fp8,0,0.3638933499654134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,128,0,1,fp8,fp8,0,0.46587733427683514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,128,0,1,float16,float16,0,0.3667733271916707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,128,0,1,float16,fp8,0,0.36634667714436847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,128,0,1,fp8,fp8,0,0.46690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,128,0,1,float16,float16,0,0.3757386604944865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,128,0,1,float16,fp8,0,0.3723893165588379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,128,0,1,float16,float16,0,0.38515734672546387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,128,0,1,float16,fp8,0,0.38310933113098145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,128,0,1,fp8,fp8,0,0.49798933664957684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,128,0,1,float16,fp8,0,0.2489173412322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,128,0,1,float16,float16,0,0.25614933172861737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,128,0,1,fp8,fp8,0,0.5034186840057373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,128,0,1,float16,float16,0,0.19147199392318726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,128,0,1,fp8,fp8,0,0.2757440010706584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,128,0,1,float16,fp8,0,0.18967467546463013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,128,0,1,float16,float16,0,0.19335466623306274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,128,0,1,fp8,fp8,0,0.2409813404083252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,128,0,1,float16,fp8,0,0.19275200366973877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,128,0,1,fp8,fp8,0,0.24267733097076416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,128,0,1,fp8,fp8,0,0.258842666943868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,128,0,1,float16,float16,0,0.1957333286603292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,128,0,1,float16,fp8,0,0.19566933314005533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,128,0,1,fp8,fp8,0,0.2544693350791931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,128,0,1,float16,fp8,0,0.20082134008407593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,128,0,1,float16,float16,0,0.19991999864578247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,128,0,1,float16,float16,0,0.13202133774757385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,128,0,1,float16,fp8,0,0.1288586656252543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,128,0,1,fp8,fp8,0,0.14618666966756186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,128,0,1,float16,float16,0,0.09856533010800679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,128,0,1,float16,fp8,0,0.09921600421269734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,128,0,1,fp8,fp8,0,0.12777599692344666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,128,0,1,float16,float16,0,0.09859733780225118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,128,0,1,float16,fp8,0,0.09938133756319682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,128,0,1,fp8,fp8,0,0.12826133767763773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,128,0,1,float16,float16,0,0.10145066181818645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,128,0,1,float16,float16,0,0.07047466437021892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,128,0,1,float16,fp8,0,0.10098133484522502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,128,0,1,fp8,fp8,0,0.13397866487503052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,128,0,1,float16,float16,0,0.10322133700052898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,128,0,1,float16,fp8,0,0.10250666737556458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,128,0,1,fp8,fp8,0,0.13507733742396036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,128,0,1,float16,fp8,0,0.06924800078074138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,128,0,1,fp8,fp8,0,0.0830026666323344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,128,0,1,float16,fp8,0,0.05570133527119955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,128,0,1,float16,float16,0,0.05453866720199585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,128,0,1,float16,fp8,0,0.05468800167242686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,128,0,1,fp8,fp8,0,0.07141333321730296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,128,0,1,float16,float16,0,0.05539200206597646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,128,0,1,fp8,fp8,0,0.07035733262697856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,128,0,1,fp8,fp8,0,0.0746666689713796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,128,0,1,float16,float16,0,0.05635733405749003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,128,0,1,float16,fp8,0,0.056746666630109154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,128,0,1,fp8,fp8,0,0.07437333464622498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,128,0,1,float16,float16,0,0.057775999108950295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,128,0,1,float16,fp8,0,0.05746666590372721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,128,0,1,float16,float16,0,0.036288000643253326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,128,0,1,float16,fp8,0,0.03745066622893015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,128,0,1,fp8,fp8,0,0.04502933224042257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,128,0,1,float16,float16,0,0.033344000577926636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,128,0,1,float16,fp8,0,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,128,0,1,fp8,fp8,0,0.04078399886687597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,128,0,1,float16,float16,0,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,128,0,1,float16,fp8,0,0.03397866586844126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,128,0,1,fp8,fp8,0,0.041834667325019836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,128,0,1,float16,float16,0,0.034101332227389015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,128,0,1,float16,fp8,0,0.03419733295838038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,128,0,1,fp8,fp8,0,0.04347200194994608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,128,0,1,float16,float16,0,0.03390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,128,0,1,float16,fp8,0,0.03418133407831192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,128,0,1,fp8,fp8,0,0.04322133461634318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,128,0,1,float16,float16,0,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,128,0,1,float16,fp8,0,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,128,0,1,float16,float16,0,0.02369066576162974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,128,0,1,fp8,fp8,0,0.03012799968322118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,128,0,1,float16,float16,0,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,128,0,1,float16,fp8,0,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,128,0,1,fp8,fp8,0,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,128,0,1,float16,fp8,0,0.02386133372783661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,128,0,1,fp8,fp8,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,128,0,1,float16,float16,0,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,128,0,1,fp8,fp8,0,0.02829866607983907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,128,0,1,float16,fp8,0,0.024480000138282776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,128,0,1,fp8,fp8,0,0.028170667588710785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,128,0,1,float16,float16,0,0.02409599969784419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,128,0,1,float16,fp8,0,0.02446399877468745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,128,0,1,float16,float16,0,0.019717333217461903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,128,0,1,float16,fp8,0,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,128,0,1,fp8,fp8,0,0.022202665607134502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,128,0,1,float16,fp8,0,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,128,0,1,float16,float16,0,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,128,0,1,float16,fp8,0,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,128,0,1,fp8,fp8,0,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,128,0,1,float16,float16,0,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,128,0,1,fp8,fp8,0,0.02094399929046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,128,0,1,float16,float16,0,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,128,0,1,fp8,fp8,0,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,128,0,1,float16,fp8,0,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,128,0,1,fp8,fp8,0,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,128,0,1,float16,float16,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,128,0,1,float16,fp8,0,0.01836266616980235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,128,0,1,float16,float16,0,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,128,0,1,float16,fp8,0,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,128,0,1,fp8,fp8,0,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,128,0,1,float16,float16,0,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,128,0,1,float16,fp8,0,0.015637333194414776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,128,0,1,fp8,fp8,0,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,128,0,1,float16,float16,0,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,128,0,1,float16,fp8,0,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,128,0,1,fp8,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,128,0,1,float16,float16,0,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,128,0,1,float16,fp8,0,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,128,0,1,fp8,fp8,0,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,128,0,1,float16,float16,0,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,128,0,1,float16,fp8,0,0.015439999600251516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,128,0,1,fp8,fp8,0,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,128,0,1,float16,float16,0,0.014682666709025701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,128,0,1,float16,fp8,0,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,128,0,1,fp8,fp8,0,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,128,0,1,float16,float16,0,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,128,0,1,float16,fp8,0,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,128,0,1,fp8,fp8,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,128,0,1,float16,float16,0,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,128,0,1,float16,fp8,0,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,128,0,1,fp8,fp8,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,128,0,1,float16,float16,0,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,128,0,1,float16,fp8,0,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,128,0,1,fp8,fp8,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,128,0,1,float16,float16,0,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,128,0,1,float16,fp8,0,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,128,0,1,fp8,fp8,0,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,128,0,1,float16,float16,0,0.29692800839742023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,128,0,1,float16,fp8,0,0.2975359956423442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,128,0,1,float16,float16,0,0.2991146643956502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,128,0,1,fp8,fp8,0,0.4049546718597412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,128,0,1,float16,fp8,0,0.30184000730514526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,128,0,1,fp8,fp8,0,0.40724798043568927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,128,0,1,float16,float16,0,0.30214933554331463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,128,0,1,float16,fp8,0,0.3017386595408122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,128,0,1,fp8,fp8,0,0.41969064871470135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,128,0,1,float16,float16,0,0.30984000364939374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,128,0,1,float16,fp8,0,0.308896005153656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,128,0,1,float16,float16,0,0.18778133392333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,128,0,1,float16,fp8,0,0.1816320021947225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,128,0,1,fp8,fp8,0,0.4241439898808797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,128,0,1,fp8,fp8,0,0.23015999794006348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,128,0,1,float16,float16,0,0.1516746679941813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,128,0,1,fp8,fp8,0,0.2121866742769877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,128,0,1,float16,fp8,0,0.1521440049012502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,128,0,1,fp8,fp8,0,0.21198934316635132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,128,0,1,float16,float16,0,0.15220800042152405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,128,0,1,float16,fp8,0,0.15344533324241638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,128,0,1,float16,float16,0,0.15465066830317178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,128,0,1,float16,fp8,0,0.15524267156918845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,128,0,1,float16,float16,0,0.1564479966958364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,128,0,1,fp8,fp8,0,0.21667200326919556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,128,0,1,float16,fp8,0,0.15665599703788757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,128,0,1,float16,float16,0,0.09628799557685852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,128,0,1,fp8,fp8,0,0.21834667523701987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,128,0,1,float16,fp8,0,0.09522133072217305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,128,0,1,fp8,fp8,0,0.12433066964149475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,128,0,1,float16,float16,0,0.08179733157157898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,128,0,1,float16,fp8,0,0.08178133269151051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,128,0,1,fp8,fp8,0,0.11106666922569275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,128,0,1,float16,float16,0,0.08296533425649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,128,0,1,float16,fp8,0,0.08224000036716461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,128,0,1,fp8,fp8,0,0.11191466450691223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,128,0,1,float16,float16,0,0.08408533533414204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,128,0,1,float16,fp8,0,0.08392533659934998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,128,0,1,fp8,fp8,0,0.11505599816640218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,128,0,1,float16,float16,0,0.083514670530955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,128,0,1,float16,fp8,0,0.08435733119646709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,128,0,1,fp8,fp8,0,0.11575466394424438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,128,0,1,float16,float16,0,0.05115733544031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,128,0,1,float16,fp8,0,0.050111999114354454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,128,0,1,float16,float16,0,0.04706133405367533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,128,0,1,fp8,fp8,0,0.06680533289909363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,128,0,1,float16,float16,0,0.04708800216515859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,128,0,1,fp8,fp8,0,0.06241600215435028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,128,0,1,float16,fp8,0,0.047050664822260536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,128,0,1,fp8,fp8,0,0.061520000298817955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,128,0,1,float16,float16,0,0.04770133395989736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,128,0,1,float16,fp8,0,0.047637333472569786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,128,0,1,float16,float16,0,0.047824000318845115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,128,0,1,fp8,fp8,0,0.06414933502674103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,128,0,1,float16,fp8,0,0.048058668772379555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,128,0,1,fp8,fp8,0,0.06355733176072438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,128,0,1,float16,fp8,0,0.04747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,128,0,1,float16,float16,0,0.03216533362865448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,128,0,1,float16,fp8,0,0.032272001107533775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,128,0,1,fp8,fp8,0,0.04091733445723852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,128,0,1,float16,float16,0,0.030271999537944794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,128,0,1,float16,fp8,0,0.030960001051425934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,128,0,1,fp8,fp8,0,0.03862933317820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,128,0,1,float16,float16,0,0.031045332551002502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,128,0,1,fp8,fp8,0,0.04008000095685323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,128,0,1,float16,fp8,0,0.03123733401298523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,128,0,1,fp8,fp8,0,0.03925333420435587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,128,0,1,float16,float16,0,0.03134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,128,0,1,float16,fp8,0,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,128,0,1,float16,float16,0,0.03173866619666418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,128,0,1,float16,fp8,0,0.031328000128269196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,128,0,1,fp8,fp8,0,0.039962666730086006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,128,0,1,float16,float16,0,0.0225600004196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,128,0,1,float16,fp8,0,0.022255999346574146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,128,0,1,fp8,fp8,0,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,128,0,1,float16,float16,0,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,128,0,1,float16,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,128,0,1,fp8,fp8,0,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,128,0,1,float16,float16,0,0.021759999295075733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,128,0,1,float16,fp8,0,0.022069332500298817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,128,0,1,fp8,fp8,0,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,128,0,1,float16,float16,0,0.02179733415444692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,128,0,1,float16,fp8,0,0.021967999637126923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,128,0,1,fp8,fp8,0,0.025807999074459076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,128,0,1,float16,float16,0,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,128,0,1,float16,fp8,0,0.021536000072956085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,128,0,1,fp8,fp8,0,0.025813333690166473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,128,0,1,float16,float16,0,0.01781333362062772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,128,0,1,float16,fp8,0,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,128,0,1,fp8,fp8,0,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,128,0,1,float16,fp8,0,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,128,0,1,float16,float16,0,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,128,0,1,fp8,fp8,0,0.020448000480731327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,128,0,1,float16,float16,0,0.024506665766239166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,128,0,1,float16,fp8,0,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,128,0,1,fp8,fp8,0,0.020495999604463577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,128,0,1,float16,float16,0,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,128,0,1,float16,fp8,0,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,128,0,1,fp8,fp8,0,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,128,0,1,float16,float16,0,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,128,0,1,float16,fp8,0,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,128,0,1,fp8,fp8,0,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,128,0,1,float16,float16,0,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,128,0,1,float16,fp8,0,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,128,0,1,fp8,fp8,0,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,128,0,1,float16,float16,0,0.014442666123310724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,128,0,1,float16,fp8,0,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,128,0,1,fp8,fp8,0,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,128,0,1,float16,float16,0,0.01458666721979777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,128,0,1,float16,fp8,0,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,128,0,1,fp8,fp8,0,0.0164533331990242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,128,0,1,float16,float16,0,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,128,0,1,float16,fp8,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,128,0,1,fp8,fp8,0,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,128,0,1,float16,float16,0,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,128,0,1,float16,fp8,0,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,128,0,1,fp8,fp8,0,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,128,0,1,float16,float16,0,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,128,0,1,float16,fp8,0,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,128,0,1,fp8,fp8,0,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,128,0,1,float16,float16,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,128,0,1,float16,fp8,0,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,128,0,1,fp8,fp8,0,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,128,0,1,float16,float16,0,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,128,0,1,float16,fp8,0,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,128,0,1,fp8,fp8,0,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,128,0,1,float16,float16,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,128,0,1,float16,fp8,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,128,0,1,fp8,fp8,0,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,128,0,1,float16,float16,0,0.013925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,128,0,1,float16,fp8,0,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,128,0,1,fp8,fp8,0,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,128,0,1,fp8,fp8,0,27.21240997314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,128,0,1,fp8,fp8,0,27.515663146972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,128,0,1,float16,float16,0,46.090484619140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,128,0,1,float16,fp8,0,45.4614512125651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,128,0,1,float16,fp8,0,47.39093017578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,128,0,1,float16,float16,0,46.90628560384115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,128,0,1,float16,fp8,0,47.0262451171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,128,0,1,float16,float16,0,47.18975321451823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,128,0,1,fp8,fp8,0,27.540809631347656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,128,0,1,fp8,fp8,0,14.49950917561849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,128,0,1,float16,float16,0,21.197855631510418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,128,0,1,float16,fp8,0,21.274213155110676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,128,0,1,fp8,fp8,0,30.368250528971355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,128,0,1,float16,float16,0,42.54430898030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,128,0,1,fp8,fp8,0,15.771733601888021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,128,0,1,float16,float16,0,20.326629638671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,128,0,1,float16,fp8,0,22.51507822672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,128,0,1,float16,fp8,0,42.450767517089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,128,0,1,fp8,fp8,0,14.004069010416666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,128,0,1,float16,float16,0,20.781349182128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,128,0,1,float16,fp8,0,21.094037373860676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,128,0,1,fp8,fp8,0,16.144575754801433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,128,0,1,float16,float16,0,20.514031728108723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,128,0,1,float16,fp8,0,21.265599568684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,128,0,1,float16,float16,0,10.71493911743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,128,0,1,float16,fp8,0,10.869738260904947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,128,0,1,fp8,fp8,0,7.469493230183919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,128,0,1,float16,float16,0,20.855108896891277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,128,0,1,fp8,fp8,0,14.517536163330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,128,0,1,float16,float16,0,10.742351531982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,128,0,1,fp8,fp8,0,7.327119827270508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,128,0,1,float16,fp8,0,22.401941935221355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,128,0,1,float16,fp8,0,11.44265619913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,128,0,1,fp8,fp8,0,7.339216232299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,128,0,1,float16,float16,0,11.711808522542318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,128,0,1,float16,fp8,0,10.408079783121744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,128,0,1,float16,float16,0,11.37875747680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,128,0,1,fp8,fp8,0,7.819552103678386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,128,0,1,float16,fp8,0,11.348101298014322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,128,0,1,fp8,fp8,0,7.443642934163411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,128,0,1,float16,float16,0,5.756314595540364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,128,0,1,fp8,fp8,0,3.8305705388387046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,128,0,1,float16,fp8,0,5.8213761647542315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,128,0,1,float16,float16,0,10.766890207926432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,128,0,1,float16,fp8,0,10.921749114990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,128,0,1,float16,float16,0,5.689626693725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,128,0,1,fp8,fp8,0,3.7380107243855796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,128,0,1,fp8,fp8,0,3.6592000325520835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,128,0,1,float16,fp8,0,5.4527842203776045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,128,0,1,float16,float16,0,5.577711741129558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,128,0,1,float16,fp8,0,5.530426661173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,128,0,1,float16,float16,0,5.393423716227214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,128,0,1,fp8,fp8,0,3.491744041442871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,128,0,1,float16,fp8,0,5.899344126383464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,128,0,1,fp8,fp8,0,3.692394574483236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,128,0,1,float16,float16,0,5.612965265909831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,128,0,1,float16,fp8,0,5.395498911539714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,128,0,1,fp8,fp8,0,17.119557698567707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,128,0,1,fp8,fp8,0,16.597984313964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,128,0,1,float16,float16,0,25.9697748819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,128,0,1,float16,fp8,0,25.92584991455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,128,0,1,float16,float16,0,25.76421356201172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,128,0,1,float16,fp8,0,26.004587809244793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,128,0,1,float16,float16,0,25.41429392496745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,128,0,1,float16,fp8,0,24.825167338053387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,128,0,1,fp8,fp8,0,16.655738830566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,128,0,1,fp8,fp8,0,8.803824106852213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,128,0,1,float16,float16,0,12.70266087849935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,128,0,1,float16,fp8,0,12.859509785970053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,128,0,1,fp8,fp8,0,16.356469472249348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,128,0,1,float16,float16,0,12.41115697224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,128,0,1,float16,float16,0,24.339007059733074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,128,0,1,fp8,fp8,0,8.181568145751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,128,0,1,float16,fp8,0,11.983189900716146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,128,0,1,float16,fp8,0,24.48621368408203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,128,0,1,fp8,fp8,0,8.472997029622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,128,0,1,float16,float16,0,12.142810821533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,128,0,1,float16,fp8,0,12.60165278116862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,128,0,1,float16,float16,0,12.320725758870443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,128,0,1,fp8,fp8,0,8.60857073465983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,128,0,1,float16,fp8,0,12.45968500773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,128,0,1,float16,float16,0,6.573098500569661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,128,0,1,fp8,fp8,0,8.983786900838217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,128,0,1,fp8,fp8,0,4.457248051961263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,128,0,1,float16,float16,0,12.856000264485678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,128,0,1,float16,fp8,0,6.788565317789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,128,0,1,float16,float16,0,6.420693079630534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,128,0,1,float16,fp8,0,12.861456553141275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,128,0,1,fp8,fp8,0,4.363381385803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,128,0,1,float16,fp8,0,6.4987837473551435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,128,0,1,fp8,fp8,0,4.321125348409017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,128,0,1,float16,float16,0,6.472351710001628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,128,0,1,float16,fp8,0,6.586090723673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,128,0,1,fp8,fp8,0,4.297189394632976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,128,0,1,float16,float16,0,6.477882385253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,128,0,1,float16,fp8,0,6.295775731404622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,128,0,1,float16,float16,0,3.316250801086426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,128,0,1,fp8,fp8,0,2.2891573905944824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,128,0,1,fp8,fp8,0,4.2729441324869795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,128,0,1,float16,fp8,0,3.1219307581583657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,128,0,1,float16,float16,0,6.429007848103841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,128,0,1,float16,fp8,0,6.428933461507161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,128,0,1,float16,float16,0,3.491397221883138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,128,0,1,fp8,fp8,0,2.1415200233459473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,128,0,1,float16,fp8,0,3.1758505503336587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,128,0,1,fp8,fp8,0,2.090655962626139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,128,0,1,float16,float16,0,3.1661812464396157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,128,0,1,float16,fp8,0,3.0893920262654624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,128,0,1,fp8,fp8,0,2.1056480407714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,128,0,1,float16,float16,0,3.1693760553995767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,128,0,1,float16,fp8,0,3.1503092447916665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,128,0,1,fp8,fp8,0,2.1732853253682456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,128,0,1,float16,float16,0,3.170821189880371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,128,0,1,float16,fp8,0,3.152485211690267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,128,0,1,fp8,fp8,0,11.703717549641928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,128,0,1,fp8,fp8,0,11.783866882324219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,128,0,1,float16,float16,0,17.177968343098957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,128,0,1,float16,fp8,0,17.938517252604168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,128,0,1,float16,float16,0,17.396458943684895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,128,0,1,float16,fp8,0,17.97228240966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,128,0,1,float16,float16,0,17.449899037679035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,128,0,1,fp8,fp8,0,12.178075154622396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,128,0,1,float16,fp8,0,16.955941518147785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,128,0,1,fp8,fp8,0,6.355045318603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,128,0,1,float16,float16,0,9.314949035644531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,128,0,1,float16,fp8,0,9.240394592285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,128,0,1,fp8,fp8,0,12.347173055013021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,128,0,1,float16,float16,0,18.023707071940105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,128,0,1,fp8,fp8,0,6.1815840403238935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,128,0,1,float16,fp8,0,8.47546132405599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,128,0,1,float16,float16,0,9.013280232747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,128,0,1,float16,fp8,0,17.45031483968099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,128,0,1,fp8,fp8,0,5.985759735107422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,128,0,1,float16,float16,0,8.658688227335611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,128,0,1,float16,fp8,0,8.763354619344076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,128,0,1,fp8,fp8,0,6.523221333821614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,128,0,1,float16,float16,0,8.735578536987305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,128,0,1,float16,fp8,0,8.75437863667806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,128,0,1,float16,float16,0,4.562874794006348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,128,0,1,float16,fp8,0,4.5425974527994795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,128,0,1,float16,float16,0,8.671557108561197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,128,0,1,fp8,fp8,0,6.019253412882487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,128,0,1,float16,fp8,0,8.673344294230143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,128,0,1,fp8,fp8,0,3.6875041325887046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,128,0,1,fp8,fp8,0,2.9245761235555015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,128,0,1,float16,float16,0,4.376864115397136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,128,0,1,fp8,fp8,0,3.0423361460367837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,128,0,1,float16,fp8,0,4.264474550882976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,128,0,1,float16,float16,0,4.267690658569336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,128,0,1,float16,fp8,0,4.380261421203613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,128,0,1,fp8,fp8,0,2.9537973403930664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,128,0,1,float16,float16,0,4.4085493087768555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,128,0,1,float16,fp8,0,4.709952036539714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,128,0,1,float16,float16,0,2.312928040822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,128,0,1,fp8,fp8,0,2.9836959838867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,128,0,1,float16,float16,0,4.4413706461588545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,128,0,1,float16,fp8,0,2.2821812629699707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,128,0,1,fp8,fp8,0,1.5636426607767742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,128,0,1,float16,fp8,0,4.694064140319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,128,0,1,float16,float16,0,2.2673120498657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,128,0,1,fp8,fp8,0,1.6036960283915203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,128,0,1,float16,fp8,0,2.2495412826538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,128,0,1,fp8,fp8,0,1.5346934000651042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,128,0,1,float16,float16,0,2.626149336496989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,128,0,1,float16,fp8,0,2.3049119313557944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,128,0,1,float16,float16,0,2.2279680569966636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,128,0,1,fp8,fp8,0,1.5834506352742512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,128,0,1,float16,fp8,0,2.331397374471029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,128,0,1,float16,float16,0,2.289210637410482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,128,0,1,fp8,fp8,0,1.515077273050944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,128,0,1,float16,fp8,0,2.37718931833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,128,0,1,fp8,fp8,0,16.76043701171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,128,0,1,fp8,fp8,0,15.921642303466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,128,0,1,float16,float16,0,23.061696370442707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,128,0,1,float16,fp8,0,23.169227600097656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,128,0,1,float16,float16,0,22.545918782552082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,128,0,1,float16,float16,0,23.014305114746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,128,0,1,float16,fp8,0,23.307579040527344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,128,0,1,float16,fp8,0,23.33275095621745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,128,0,1,fp8,fp8,0,15.642752329508463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,128,0,1,fp8,fp8,0,8.864362716674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,128,0,1,float16,float16,0,11.677419026692709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,128,0,1,float16,float16,0,11.96063486735026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,128,0,1,float16,fp8,0,12.043807983398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,128,0,1,fp8,fp8,0,15.834602355957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,128,0,1,float16,float16,0,22.938608805338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,128,0,1,fp8,fp8,0,7.867754618326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,128,0,1,float16,fp8,0,11.489823659261068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,128,0,1,fp8,fp8,0,7.9602508544921875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,128,0,1,float16,fp8,0,23.351839701334637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,128,0,1,float16,float16,0,11.690831502278646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,128,0,1,float16,fp8,0,11.945200602213541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,128,0,1,fp8,fp8,0,7.688586552937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,128,0,1,float16,float16,0,11.867088317871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,128,0,1,float16,fp8,0,11.88357925415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,128,0,1,float16,float16,0,6.122426350911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,128,0,1,fp8,fp8,0,4.0607039133707685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,128,0,1,fp8,fp8,0,8.045733133951822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,128,0,1,float16,fp8,0,6.134063720703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,128,0,1,float16,float16,0,11.771717071533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,128,0,1,float16,fp8,0,12.248245239257812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,128,0,1,float16,float16,0,6.068127950032552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,128,0,1,float16,fp8,0,5.763925552368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,128,0,1,fp8,fp8,0,3.9751466115315757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,128,0,1,fp8,fp8,0,3.9887253443400064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,128,0,1,float16,float16,0,6.331989288330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,128,0,1,float16,fp8,0,5.801727930704753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,128,0,1,fp8,fp8,0,3.897856076558431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,128,0,1,float16,float16,0,5.692698796590169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,128,0,1,float16,fp8,0,5.713226954142253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,128,0,1,fp8,fp8,0,4.053685188293457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,128,0,1,float16,float16,0,3.174799919128418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,128,0,1,float16,float16,0,5.758031845092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,128,0,1,float16,fp8,0,5.753615697224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,128,0,1,fp8,fp8,0,2.0671146710713706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,128,0,1,float16,fp8,0,3.03166929880778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,128,0,1,float16,float16,0,2.9031041463216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,128,0,1,fp8,fp8,0,2.029637336730957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,128,0,1,float16,fp8,0,3.15010134379069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,128,0,1,float16,float16,0,2.8784214655558267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,128,0,1,fp8,fp8,0,1.8963839213053386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,128,0,1,float16,fp8,0,2.8699467976888022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,128,0,1,float16,float16,0,2.8476905822753906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,128,0,1,fp8,fp8,0,2.0590240160624185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,128,0,1,float16,float16,0,1.4966452916463215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,128,0,1,float16,fp8,0,2.9462238947550454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,128,0,1,fp8,fp8,0,1.9087732632954915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,128,0,1,float16,float16,0,2.894517262776693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,128,0,1,fp8,fp8,0,1.1163520018259685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,128,0,1,float16,fp8,0,2.8713865280151367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,128,0,1,float16,fp8,0,1.4769493738810222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,128,0,1,fp8,fp8,0,1.0297386646270752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,128,0,1,float16,float16,0,1.4543733596801758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,128,0,1,fp8,fp8,0,1.0354560216267903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,128,0,1,float16,fp8,0,1.443818728129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,128,0,1,float16,float16,0,1.4730879465738933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,128,0,1,float16,fp8,0,1.4351627031962078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,128,0,1,float16,float16,0,1.536901315053304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,128,0,1,float16,fp8,0,1.496869405110677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,128,0,1,fp8,fp8,0,1.0304959615071614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,128,0,1,fp8,fp8,0,1.0444213549296062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,128,0,1,float16,float16,0,1.4497547149658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,128,0,1,float16,fp8,0,1.482319990793864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,128,0,1,fp8,fp8,0,9.300282796223959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,128,0,1,fp8,fp8,0,9.462511698404947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,128,0,1,float16,fp8,0,13.07690684000651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,128,0,1,float16,float16,0,13.753439585367838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,128,0,1,float16,fp8,0,13.71294911702474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,128,0,1,float16,float16,0,13.955317179361979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,128,0,1,float16,float16,0,13.734176635742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,128,0,1,float16,fp8,0,13.346832275390625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,128,0,1,fp8,fp8,0,9.66588274637858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,128,0,1,fp8,fp8,0,5.1034345626831055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,128,0,1,float16,float16,0,7.395365397135417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,128,0,1,float16,fp8,0,7.2450612386067705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,128,0,1,float16,float16,0,7.014464060465495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,128,0,1,fp8,fp8,0,9.618821461995443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,128,0,1,float16,float16,0,13.52563730875651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,128,0,1,fp8,fp8,0,4.67523733774821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,128,0,1,float16,fp8,0,13.158058166503906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,128,0,1,float16,fp8,0,6.840496063232422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,128,0,1,fp8,fp8,0,4.997605323791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,128,0,1,float16,float16,0,6.804416020711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,128,0,1,float16,fp8,0,6.81993039449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,128,0,1,fp8,fp8,0,4.829173405965169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,128,0,1,float16,float16,0,7.07420285542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,128,0,1,float16,fp8,0,6.928789138793945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,128,0,1,fp8,fp8,0,4.94978682200114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,128,0,1,float16,float16,0,3.65611203511556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,128,0,1,float16,fp8,0,3.6922079722086587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,128,0,1,fp8,fp8,0,2.562405268351237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,128,0,1,float16,fp8,0,6.812154769897461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,128,0,1,float16,float16,0,8.361749649047852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,128,0,1,float16,float16,0,3.4052534103393555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,128,0,1,fp8,fp8,0,2.3170666694641113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,128,0,1,float16,fp8,0,3.369434674580892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,128,0,1,fp8,fp8,0,2.2965760231018066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,128,0,1,float16,fp8,0,3.410837491353353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,128,0,1,float16,float16,0,3.535781224568685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,128,0,1,fp8,fp8,0,2.3834826151529946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,128,0,1,float16,float16,0,3.2818454106648765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,128,0,1,float16,fp8,0,3.4305973052978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,128,0,1,float16,float16,0,1.7561225891113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,128,0,1,fp8,fp8,0,2.46943998336792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,128,0,1,float16,fp8,0,1.7325654029846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,128,0,1,float16,fp8,0,3.3798240025838218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,128,0,1,float16,float16,0,3.595088005065918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,128,0,1,fp8,fp8,0,1.3217066923777263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,128,0,1,float16,float16,0,1.7065653800964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,128,0,1,fp8,fp8,0,1.2052640120188396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,128,0,1,float16,fp8,0,1.773695945739746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,128,0,1,fp8,fp8,0,1.1627466678619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,128,0,1,float16,float16,0,1.654688040415446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,128,0,1,float16,fp8,0,1.6199572881062825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,128,0,1,fp8,fp8,0,1.1671840349833171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,128,0,1,float16,float16,0,1.6654292742411296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,128,0,1,float16,fp8,0,1.653824011484782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,128,0,1,float16,float16,0,1.773706595102946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,128,0,1,float16,float16,0,0.9075307051340739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,128,0,1,fp8,fp8,0,1.1739839712778728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,128,0,1,fp8,fp8,0,0.6770453453063965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,128,0,1,float16,fp8,0,0.9447786808013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,128,0,1,float16,fp8,0,0.8968266646067301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,128,0,1,fp8,fp8,0,0.7151947021484375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,128,0,1,float16,float16,0,0.8794026374816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,128,0,1,float16,fp8,0,1.8046026229858398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,128,0,1,float16,float16,0,0.8831253051757812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,128,0,1,float16,fp8,0,0.8742667039235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,128,0,1,fp8,fp8,0,0.6452213525772095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,128,0,1,float16,float16,0,0.8985066413879395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,128,0,1,fp8,fp8,0,0.6521066824595133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,128,0,1,float16,fp8,0,0.9015786647796631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,128,0,1,float16,float16,0,0.878661314646403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,128,0,1,float16,fp8,0,0.8945013682047526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,128,0,1,fp8,fp8,0,0.6490986744562784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,128,0,1,fp8,fp8,0,9.279167811075846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,128,0,1,fp8,fp8,0,8.978480021158854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,128,0,1,float16,float16,0,12.440687815348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,128,0,1,float16,fp8,0,12.881460825602213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,128,0,1,float16,fp8,0,12.755087534586588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,128,0,1,float16,float16,0,13.093461354573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,128,0,1,float16,float16,0,12.837659200032553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,128,0,1,fp8,fp8,0,9.456186930338541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,128,0,1,fp8,fp8,0,5.094389279683431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,128,0,1,float16,float16,0,7.144954681396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,128,0,1,float16,fp8,0,7.164672215779622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,128,0,1,float16,fp8,0,12.73199462890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,128,0,1,fp8,fp8,0,9.496122360229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,128,0,1,float16,float16,0,12.86734390258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,128,0,1,fp8,fp8,0,4.705050786336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,128,0,1,float16,fp8,0,13.36635716756185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,128,0,1,float16,float16,0,6.692517598470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,128,0,1,float16,fp8,0,6.511018753051758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,128,0,1,fp8,fp8,0,4.850165367126465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,128,0,1,float16,float16,0,6.665493647257487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,128,0,1,float16,fp8,0,6.379434585571289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,128,0,1,float16,float16,0,6.627290725708008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,128,0,1,fp8,fp8,0,4.781008084615071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,128,0,1,float16,float16,0,3.66590944925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,128,0,1,float16,fp8,0,3.403349240620931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,128,0,1,fp8,fp8,0,4.933578809102376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,128,0,1,float16,fp8,0,7.058597564697266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,128,0,1,fp8,fp8,0,2.789370536804199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,128,0,1,float16,float16,0,7.014607747395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,128,0,1,float16,fp8,0,7.129093170166016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,128,0,1,fp8,fp8,0,2.158506711324056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,128,0,1,float16,float16,0,3.345055898030599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,128,0,1,float16,fp8,0,3.52076784769694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,128,0,1,fp8,fp8,0,2.6808106104532876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,128,0,1,float16,float16,0,3.2621386845906577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,128,0,1,float16,fp8,0,3.262202580769857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,128,0,1,float16,float16,0,3.267610549926758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,128,0,1,float16,fp8,0,3.1968425114949546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,128,0,1,fp8,fp8,0,1.4156427383422852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,128,0,1,float16,float16,0,1.6556533177693684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,128,0,1,float16,fp8,0,1.695759932200114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,128,0,1,fp8,fp8,0,2.1885120073954263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,128,0,1,float16,float16,0,1.6256052652994792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,128,0,1,fp8,fp8,0,2.2673919995625815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,128,0,1,float16,float16,0,3.206517219543457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,128,0,1,float16,fp8,0,3.3363733291625977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,128,0,1,fp8,fp8,0,1.1385707060496013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,128,0,1,float16,fp8,0,1.5070026715596516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,128,0,1,float16,float16,0,1.5221439997355144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,128,0,1,float16,fp8,0,1.6500693957010906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,128,0,1,fp8,fp8,0,1.1212586561838787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,128,0,1,fp8,fp8,0,1.1419413089752197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,128,0,1,float16,float16,0,1.5131999651590984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,128,0,1,float16,fp8,0,1.5287200609842937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,128,0,1,float16,fp8,0,1.5390772819519043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,128,0,1,fp8,fp8,0,1.1576426823933919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,128,0,1,float16,float16,0,0.8628479639689127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,128,0,1,fp8,fp8,0,0.6696053345998129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,128,0,1,float16,fp8,0,0.8559786478678385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,128,0,1,float16,float16,0,1.5334186553955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,128,0,1,float16,float16,0,0.7954133351643881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,128,0,1,float16,fp8,0,0.8094666798909506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,128,0,1,fp8,fp8,0,0.6149653196334839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,128,0,1,fp8,fp8,0,0.6086986859639486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,128,0,1,fp8,fp8,0,0.6171520153681437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,128,0,1,float16,fp8,0,0.8236533006032308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,128,0,1,float16,float16,0,0.8228159745534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,128,0,1,float16,float16,0,0.804474671681722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,128,0,1,float16,fp8,0,0.8059999942779541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,128,0,1,float16,float16,0,0.8160639603932699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,128,0,1,float16,float16,0,0.47494932015736896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,128,0,1,fp8,fp8,0,0.6120320161183676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,128,0,1,float16,fp8,0,0.47838401794433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,128,0,1,float16,fp8,0,0.830677350362142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,128,0,1,fp8,fp8,0,0.3678453365961711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,128,0,1,float16,float16,0,0.45256535212198895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,128,0,1,fp8,fp8,0,0.34178133805592853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,128,0,1,float16,fp8,0,0.4500906864802043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,128,0,1,float16,float16,0,0.4611519972483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,128,0,1,float16,fp8,0,0.45196266969045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,128,0,1,fp8,fp8,0,0.3366026480992635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,128,0,1,float16,float16,0,0.4656906525293986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,128,0,1,float16,fp8,0,0.4652373393376668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,128,0,1,fp8,fp8,0,0.33162667353947956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,128,0,1,float16,float16,0,0.46095999081929523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,128,0,1,float16,fp8,0,0.4546560049057007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,128,0,1,fp8,fp8,0,0.33985598882039386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,128,0,1,fp8,fp8,0,5.616282780965169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,128,0,1,float16,float16,0,7.749370574951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,128,0,1,fp8,fp8,0,5.566698710123698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,128,0,1,float16,fp8,0,7.851978937784831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,128,0,1,float16,float16,0,7.822538375854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,128,0,1,float16,fp8,0,7.82151476542155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,128,0,1,float16,fp8,0,7.881109237670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,128,0,1,float16,float16,0,7.981376012166341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,128,0,1,fp8,fp8,0,5.729248046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,128,0,1,fp8,fp8,0,3.116602579752604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,128,0,1,float16,float16,0,4.27950922648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,128,0,1,float16,fp8,0,4.268682797749837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,128,0,1,fp8,fp8,0,5.763338724772136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,128,0,1,float16,float16,0,7.870693206787109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,128,0,1,float16,fp8,0,7.78764279683431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,128,0,1,fp8,fp8,0,2.7326294581095376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,128,0,1,float16,fp8,0,3.7826560338338218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,128,0,1,float16,float16,0,3.793365478515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,128,0,1,float16,float16,0,3.684704144795736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,128,0,1,float16,fp8,0,3.777322769165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,128,0,1,fp8,fp8,0,2.7802934646606445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,128,0,1,float16,float16,0,3.810426712036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,128,0,1,fp8,fp8,0,2.86734930674235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,128,0,1,float16,fp8,0,4.003546714782715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,128,0,1,float16,fp8,0,2.0159734090169272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,128,0,1,float16,float16,0,2.0301440556844077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,128,0,1,fp8,fp8,0,2.765082677205404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,128,0,1,fp8,fp8,0,1.5789333979288738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,128,0,1,float16,fp8,0,3.8887627919514975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,128,0,1,float16,float16,0,3.7869866689046225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,128,0,1,float16,float16,0,1.8710026741027832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,128,0,1,float16,fp8,0,1.8809812863667805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,128,0,1,fp8,fp8,0,1.3785173098246257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,128,0,1,float16,float16,0,1.848522663116455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,128,0,1,fp8,fp8,0,1.3878560066223145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,128,0,1,float16,fp8,0,1.8494666417439778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,128,0,1,fp8,fp8,0,1.4613547325134277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,128,0,1,float16,float16,0,1.8860799471537273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,128,0,1,float16,fp8,0,1.8679359753926594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,128,0,1,float16,float16,0,1.9102719624837239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,128,0,1,float16,float16,0,1.0447999636332195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,128,0,1,float16,fp8,0,1.835978666941325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,128,0,1,float16,fp8,0,1.057360013326009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,128,0,1,fp8,fp8,0,0.8301119804382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,128,0,1,fp8,fp8,0,1.4315733909606934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,128,0,1,float16,float16,0,0.9351680278778076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,128,0,1,float16,fp8,0,0.9695040384928385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,128,0,1,fp8,fp8,0,0.7351679801940918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,128,0,1,fp8,fp8,0,0.7424053351084391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,128,0,1,float16,float16,0,0.9476213455200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,128,0,1,float16,fp8,0,0.9781546592712402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,128,0,1,fp8,fp8,0,0.7209493319193522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,128,0,1,float16,float16,0,0.9591360092163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,128,0,1,float16,fp8,0,0.9559360345204672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,128,0,1,float16,float16,0,0.9668107032775879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,128,0,1,float16,fp8,0,0.9689280192057291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,128,0,1,fp8,fp8,0,0.7450239658355713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,128,0,1,float16,float16,0,0.548527995745341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,128,0,1,fp8,fp8,0,0.43166399002075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,128,0,1,float16,fp8,0,0.5584693352381388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,128,0,1,float16,float16,0,0.5131680170694987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,128,0,1,float16,fp8,0,0.5113386710484823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,128,0,1,fp8,fp8,0,0.4056053161621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,128,0,1,fp8,fp8,0,0.3999040126800537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,128,0,1,float16,fp8,0,0.5140373309453329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,128,0,1,float16,float16,0,0.5111519893010458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,128,0,1,float16,float16,0,0.5064586798350016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,128,0,1,fp8,fp8,0,0.40655465920766193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,128,0,1,float16,fp8,0,0.5169279972712199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,128,0,1,float16,float16,0,0.5153013467788696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,128,0,1,float16,float16,0,0.3102560043334961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,128,0,1,float16,fp8,0,0.5206826527913412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,128,0,1,fp8,fp8,0,0.40567465623219806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,128,0,1,fp8,fp8,0,0.2403786579767863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,128,0,1,float16,fp8,0,0.3146666685740153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,128,0,1,float16,float16,0,0.3023146589597066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,128,0,1,fp8,fp8,0,0.22459733486175537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,128,0,1,float16,fp8,0,0.30103999376296997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,128,0,1,float16,fp8,0,0.3039360046386719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,128,0,1,float16,float16,0,0.2999200026194255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,128,0,1,fp8,fp8,0,0.22355733315149942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,128,0,1,float16,fp8,0,0.30211732784907025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,128,0,1,float16,float16,0,0.2959146698315938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,128,0,1,fp8,fp8,0,0.22745599349339804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,128,0,1,fp8,fp8,0,0.22645866870880127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,128,0,1,float16,float16,0,0.3036800026893616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,128,0,1,float16,fp8,0,0.305567999680837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,128,0,1,fp8,fp8,0,5.893125534057617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,128,0,1,fp8,fp8,0,6.035839716593425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,128,0,1,float16,float16,0,7.82917849222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,128,0,1,float16,fp8,0,7.8356583913167315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,128,0,1,float16,float16,0,7.923456192016602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,128,0,1,float16,fp8,0,8.091429392496744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,128,0,1,float16,float16,0,8.064624150594076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,128,0,1,float16,fp8,0,7.8889815012613935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,128,0,1,fp8,fp8,0,6.016208012898763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,128,0,1,fp8,fp8,0,3.3635787963867188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,128,0,1,float16,float16,0,4.389360109965007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,128,0,1,float16,float16,0,3.707829475402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,128,0,1,float16,fp8,0,4.33246390024821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,128,0,1,fp8,fp8,0,6.156911849975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,128,0,1,float16,float16,0,8.293605168660482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,128,0,1,float16,fp8,0,8.182624181111654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,128,0,1,fp8,fp8,0,2.899615923563639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,128,0,1,float16,fp8,0,3.75111452738444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,128,0,1,fp8,fp8,0,3.0154240926106772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,128,0,1,float16,fp8,0,3.8006772994995117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,128,0,1,float16,float16,0,3.9104747772216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,128,0,1,float16,float16,0,3.9256906509399414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,128,0,1,fp8,fp8,0,2.9963467915852866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,128,0,1,float16,fp8,0,3.8504746754964194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,128,0,1,float16,float16,0,3.951274553934733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,128,0,1,fp8,fp8,0,1.6950507164001465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,128,0,1,float16,fp8,0,3.9117654164632163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,128,0,1,float16,fp8,0,2.14084259668986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,128,0,1,float16,float16,0,2.165653387705485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,128,0,1,fp8,fp8,0,3.003514607747396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,128,0,1,float16,float16,0,1.8688480059305828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,128,0,1,float16,fp8,0,1.837173302968343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,128,0,1,fp8,fp8,0,1.4399040540059407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,128,0,1,float16,float16,0,1.8561333020528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,128,0,1,float16,fp8,0,1.8591465950012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,128,0,1,fp8,fp8,0,1.4834507306416829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,128,0,1,float16,float16,0,1.859663963317871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,128,0,1,fp8,fp8,0,1.4666399955749512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,128,0,1,float16,fp8,0,1.9389813741048176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,128,0,1,float16,float16,0,1.9155839284261067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,128,0,1,float16,float16,0,1.094000021616618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,128,0,1,fp8,fp8,0,0.8913066387176514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,128,0,1,float16,fp8,0,1.0816960334777832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,128,0,1,fp8,fp8,0,1.5144319534301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,128,0,1,float16,fp8,0,1.9250772794087727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,128,0,1,float16,float16,0,0.9624799887339274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,128,0,1,float16,fp8,0,0.969925324122111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,128,0,1,fp8,fp8,0,0.7592746416727701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,128,0,1,float16,float16,0,0.9400373299916586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,128,0,1,fp8,fp8,0,0.7601706981658936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,128,0,1,float16,fp8,0,0.9513599872589111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,128,0,1,fp8,fp8,0,0.7806453704833984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,128,0,1,float16,float16,0,0.9399786790211996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,128,0,1,float16,fp8,0,0.961674690246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,128,0,1,float16,float16,0,0.9590613047281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,128,0,1,fp8,fp8,0,0.774346669514974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,128,0,1,float16,fp8,0,0.9659039974212646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,128,0,1,float16,float16,0,0.5641066630681356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,128,0,1,float16,fp8,0,0.5648159980773926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,128,0,1,fp8,fp8,0,0.4734506607055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,128,0,1,float16,float16,0,0.49934931596120197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,128,0,1,float16,fp8,0,0.5082079966862997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,128,0,1,fp8,fp8,0,0.4097706476847331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,128,0,1,float16,float16,0,0.499232014020284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,128,0,1,float16,fp8,0,0.5064533154169718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,128,0,1,fp8,fp8,0,0.41123199462890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,128,0,1,float16,float16,0,0.5040586789449056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,128,0,1,float16,fp8,0,0.5049599806467692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,128,0,1,fp8,fp8,0,0.41281068325042725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,128,0,1,float16,float16,0,0.5074079831441244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,128,0,1,float16,fp8,0,0.5122400124867758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,128,0,1,fp8,fp8,0,0.41680534680684406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,128,0,1,float16,float16,0,0.3047893246014913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,128,0,1,float16,fp8,0,0.3052106698354085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,128,0,1,fp8,fp8,0,0.25411732991536456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,128,0,1,float16,float16,0,0.2807893355687459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,128,0,1,float16,float16,0,0.2792159914970398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,128,0,1,float16,fp8,0,0.2836640079816182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,128,0,1,fp8,fp8,0,0.21997867027918497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,128,0,1,float16,fp8,0,0.28335465987523395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,128,0,1,fp8,fp8,0,0.21915733814239502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,128,0,1,float16,float16,0,0.2829280098279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,128,0,1,float16,fp8,0,0.2840106685956319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,128,0,1,fp8,fp8,0,0.21825067202250162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,128,0,1,float16,float16,0,0.28598399957021076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,128,0,1,float16,fp8,0,0.28895999987920123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,128,0,1,fp8,fp8,0,0.22403732935587564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,128,0,1,float16,float16,0,0.16846932967503866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,128,0,1,float16,fp8,0,0.17061332861582437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,128,0,1,fp8,fp8,0,0.1465120017528534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,128,0,1,float16,float16,0,0.1564959983030955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,128,0,1,float16,fp8,0,0.15631999572118124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,128,0,1,fp8,fp8,0,0.13212266564369202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,128,0,1,float16,float16,0,0.1567039986451467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,128,0,1,float16,fp8,0,0.15758933623631796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,128,0,1,fp8,fp8,0,0.1320746640364329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,128,0,1,float16,float16,0,0.1574079990386963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,128,0,1,float16,fp8,0,0.15818666418393454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,128,0,1,fp8,fp8,0,0.13291200002034506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,128,0,1,float16,float16,0,0.15637866655985513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,128,0,1,float16,fp8,0,0.1600320041179657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,128,0,1,fp8,fp8,0,0.13366400202115378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,128,0,1,fp8,fp8,0,3.9839626948038735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,128,0,1,fp8,fp8,0,3.9389333724975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,128,0,1,float16,float16,0,4.99564266204834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,128,0,1,float16,fp8,0,4.995770772298177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,128,0,1,float16,float16,0,4.8709761301676435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,128,0,1,float16,fp8,0,4.994751930236816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,128,0,1,float16,float16,0,5.00979741414388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,128,0,1,float16,fp8,0,5.162927945454915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,128,0,1,fp8,fp8,0,3.965087890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,128,0,1,fp8,fp8,0,2.304170608520508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,128,0,1,float16,float16,0,2.8455146153767905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,128,0,1,float16,float16,0,2.4335840543111167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,128,0,1,float16,fp8,0,2.786106745402018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,128,0,1,fp8,fp8,0,4.022213300069173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,128,0,1,float16,float16,0,5.091461181640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,128,0,1,float16,fp8,0,5.208463986714681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,128,0,1,fp8,fp8,0,1.9441760381062825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,128,0,1,float16,fp8,0,2.402458667755127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,128,0,1,fp8,fp8,0,1.9870400428771973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,128,0,1,float16,float16,0,2.3934079806009927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,128,0,1,float16,fp8,0,2.4808640480041504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,128,0,1,float16,float16,0,2.453013261159261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,128,0,1,float16,fp8,0,2.42520538965861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,128,0,1,fp8,fp8,0,1.9712586402893066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,128,0,1,float16,float16,0,1.4305493036905925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,128,0,1,float16,fp8,0,1.4116533597310383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,128,0,1,fp8,fp8,0,2.016117254892985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,128,0,1,float16,float16,0,2.5253599484761557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,128,0,1,float16,fp8,0,2.5275306701660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,128,0,1,fp8,fp8,0,1.1631413300832112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,128,0,1,float16,float16,0,1.1598186492919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,128,0,1,float16,fp8,0,1.187557299931844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,128,0,1,fp8,fp8,0,0.9897226492563883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,128,0,1,float16,float16,0,1.1566239992777507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,128,0,1,float16,fp8,0,1.1927786668141682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,128,0,1,fp8,fp8,0,1.011071999867757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,128,0,1,fp8,fp8,0,1.0064319769541423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,128,0,1,float16,float16,0,1.180741310119629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,128,0,1,float16,fp8,0,1.2005493640899658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,128,0,1,float16,float16,0,1.197002649307251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,128,0,1,float16,float16,0,0.7287306785583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,128,0,1,float16,fp8,0,0.7157066663106283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,128,0,1,fp8,fp8,0,1.0239466826121013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,128,0,1,float16,fp8,0,1.2064159711201985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,128,0,1,fp8,fp8,0,0.6086826721827189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,128,0,1,float16,float16,0,0.6048640012741089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,128,0,1,float16,fp8,0,0.6119466622670492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,128,0,1,fp8,fp8,0,0.5172533194224039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,128,0,1,float16,float16,0,0.6112533410390218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,128,0,1,fp8,fp8,0,0.520799994468689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,128,0,1,float16,fp8,0,0.6174453496932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,128,0,1,float16,float16,0,0.6149599949518839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,128,0,1,float16,fp8,0,0.6154346863428751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,128,0,1,fp8,fp8,0,0.5223413308461508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,128,0,1,float16,float16,0,0.6235680182774862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,128,0,1,float16,fp8,0,0.6270773410797119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,128,0,1,float16,float16,0,0.37987200419108075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,128,0,1,float16,fp8,0,0.3760960102081299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,128,0,1,fp8,fp8,0,0.5340640147527059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,128,0,1,fp8,fp8,0,0.3227573235829671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,128,0,1,float16,fp8,0,0.33022934198379517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,128,0,1,fp8,fp8,0,0.28153600295384723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,128,0,1,float16,float16,0,0.3282080094019572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,128,0,1,float16,float16,0,0.3300480047861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,128,0,1,fp8,fp8,0,0.284986674785614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,128,0,1,float16,fp8,0,0.3356533447901408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,128,0,1,float16,float16,0,0.33214400211970013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,128,0,1,fp8,fp8,0,0.29073067506154376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,128,0,1,float16,fp8,0,0.33266133069992065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,128,0,1,fp8,fp8,0,0.2844533324241638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,128,0,1,float16,float16,0,0.3359413146972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,128,0,1,float16,fp8,0,0.3396960099538167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,128,0,1,float16,float16,0,0.20948266983032227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,128,0,1,float16,fp8,0,0.2108853260676066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,128,0,1,fp8,fp8,0,0.17575999101003012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,128,0,1,float16,float16,0,0.18787733713785806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,128,0,1,float16,fp8,0,0.1885706583658854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,128,0,1,float16,float16,0,0.19153600931167603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,128,0,1,fp8,fp8,0,0.1525759994983673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,128,0,1,float16,fp8,0,0.19274665911992392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,128,0,1,float16,float16,0,0.19016534090042114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,128,0,1,float16,fp8,0,0.19166932503382364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,128,0,1,float16,fp8,0,0.1951786677042643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,128,0,1,fp8,fp8,0,0.15306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,128,0,1,fp8,fp8,0,0.1544266641139984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,128,0,1,float16,float16,0,0.19412267208099365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,128,0,1,fp8,fp8,0,0.1573919951915741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,128,0,1,float16,float16,0,0.11707733074824016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,128,0,1,float16,fp8,0,0.11943466464678447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,128,0,1,fp8,fp8,0,0.10545600454012553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,128,0,1,float16,float16,0,0.10805867115656535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,128,0,1,float16,fp8,0,0.1092800001303355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,128,0,1,fp8,fp8,0,0.09520533680915833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,128,0,1,float16,fp8,0,0.10778133074442546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,128,0,1,float16,float16,0,0.10955199599266052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,128,0,1,float16,fp8,0,0.11058666308720906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,128,0,1,fp8,fp8,0,0.09486400087674458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,128,0,1,float16,float16,0,0.10866133371988933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,128,0,1,fp8,fp8,0,0.09550933043162028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,128,0,1,float16,float16,0,0.10883200168609619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,128,0,1,float16,fp8,0,0.1106719970703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,128,0,1,fp8,fp8,0,0.0958079993724823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,128,0,1,fp8,fp8,0,4.272261301676433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,128,0,1,fp8,fp8,0,4.274229367574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,128,0,1,float16,float16,0,5.015578587849935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,128,0,1,float16,fp8,0,4.987589200337728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,128,0,1,float16,float16,0,5.171504020690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,128,0,1,float16,fp8,0,4.9997866948445635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,128,0,1,float16,float16,0,5.2518666585286455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,128,0,1,float16,fp8,0,5.087765375773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,128,0,1,float16,float16,0,2.443818728129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,128,0,1,fp8,fp8,0,2.7290827433268228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,128,0,1,fp8,fp8,0,4.345770517985026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,128,0,1,float16,float16,0,3.2788108189900718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,128,0,1,float16,fp8,0,3.1415627797444663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,128,0,1,fp8,fp8,0,4.449562708536784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,128,0,1,float16,float16,0,5.360298792521159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,128,0,1,float16,fp8,0,5.48312505086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,128,0,1,fp8,fp8,0,2.0901759465535483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,128,0,1,float16,fp8,0,2.48907740910848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,128,0,1,fp8,fp8,0,2.1735626856486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,128,0,1,float16,float16,0,2.447450637817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,128,0,1,float16,fp8,0,2.54694398244222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,128,0,1,float16,float16,0,2.5135679244995117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,128,0,1,float16,fp8,0,2.5481600761413574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,128,0,1,fp8,fp8,0,2.189546744028727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,128,0,1,float16,float16,0,1.6338079770406086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,128,0,1,float16,float16,0,2.6715145111083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,128,0,1,fp8,fp8,0,2.285615921020508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,128,0,1,float16,fp8,0,2.6813653310139975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,128,0,1,float16,fp8,0,1.5449546178181965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,128,0,1,float16,float16,0,1.2268906434377034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,128,0,1,fp8,fp8,0,1.3532800674438477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,128,0,1,float16,fp8,0,1.2421546777089436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,128,0,1,fp8,fp8,0,1.0480159918467205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,128,0,1,float16,float16,0,1.2485333283742268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,128,0,1,float16,fp8,0,1.2554186979929607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,128,0,1,fp8,fp8,0,1.0675146579742432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,128,0,1,float16,float16,0,1.279263973236084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,128,0,1,fp8,fp8,0,1.1205493609110515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,128,0,1,float16,fp8,0,1.247098684310913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,128,0,1,float16,float16,0,1.2903199990590413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,128,0,1,float16,fp8,0,1.2857706546783447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,128,0,1,fp8,fp8,0,1.133898655573527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,128,0,1,float16,float16,0,0.8056213061014811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,128,0,1,fp8,fp8,0,0.6781333287556967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,128,0,1,float16,float16,0,0.6356213490168253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,128,0,1,float16,fp8,0,0.7881600062052408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,128,0,1,fp8,fp8,0,0.5400906801223755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,128,0,1,float16,fp8,0,0.6313653389612833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,128,0,1,float16,float16,0,0.636517326037089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,128,0,1,float16,fp8,0,0.6267733176549276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,128,0,1,fp8,fp8,0,0.5390880107879639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,128,0,1,float16,float16,0,0.6480426788330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,128,0,1,fp8,fp8,0,0.5518879890441895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,128,0,1,float16,fp8,0,0.6407893498738607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,128,0,1,float16,float16,0,0.6598933140436808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,128,0,1,float16,fp8,0,0.6571360031763712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,128,0,1,float16,float16,0,0.42193599541982013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,128,0,1,fp8,fp8,0,0.5580319960912069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,128,0,1,float16,fp8,0,0.4084159930547078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,128,0,1,fp8,fp8,0,0.3476853370666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,128,0,1,float16,fp8,0,0.32105066378911334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,128,0,1,fp8,fp8,0,0.2779253323872884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,128,0,1,float16,float16,0,0.3202400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,128,0,1,fp8,fp8,0,0.2752000093460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,128,0,1,float16,float16,0,0.3261813322703044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,128,0,1,float16,fp8,0,0.32733333110809326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,128,0,1,float16,float16,0,0.3376479943593343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,128,0,1,fp8,fp8,0,0.2834933400154114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,128,0,1,float16,fp8,0,0.335098663965861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,128,0,1,float16,float16,0,0.34089601039886475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,128,0,1,float16,fp8,0,0.336357315381368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,128,0,1,fp8,fp8,0,0.28538666168848675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,128,0,1,float16,float16,0,0.22332799434661865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,128,0,1,float16,fp8,0,0.21827733516693115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,128,0,1,fp8,fp8,0,0.18276800711949667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,128,0,1,float16,float16,0,0.17278399070103964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,128,0,1,float16,fp8,0,0.17499200503031412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,128,0,1,fp8,fp8,0,0.13743999600410461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,128,0,1,float16,fp8,0,0.1807039976119995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,128,0,1,float16,float16,0,0.17458132902781168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,128,0,1,float16,fp8,0,0.1737066706021627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,128,0,1,fp8,fp8,0,0.13762666781743368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,128,0,1,float16,float16,0,0.1811359922091166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,128,0,1,fp8,fp8,0,0.13936000068982443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,128,0,1,fp8,fp8,0,0.09692266583442688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,128,0,1,float16,float16,0,0.18418665726979574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,128,0,1,float16,fp8,0,0.18338666359583536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,128,0,1,fp8,fp8,0,0.1437546710173289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,128,0,1,float16,float16,0,0.12487467130025227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,128,0,1,float16,fp8,0,0.12112533052762349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,128,0,1,float16,float16,0,0.0941973328590393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,128,0,1,float16,fp8,0,0.09335466225941975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,128,0,1,fp8,fp8,0,0.0773226668437322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,128,0,1,float16,float16,0,0.0942133367061615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,128,0,1,float16,fp8,0,0.09447999795277913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,128,0,1,fp8,fp8,0,0.07820799946784973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,128,0,1,float16,float16,0,0.0944106678167979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,128,0,1,float16,fp8,0,0.0956106682618459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,128,0,1,fp8,fp8,0,0.0800906668106715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,128,0,1,float16,float16,0,0.09748799602190654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,128,0,1,float16,fp8,0,0.09768533706665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,128,0,1,fp8,fp8,0,0.08185066779454549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,128,0,1,float16,float16,0,0.06434666613737743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,128,0,1,float16,fp8,0,0.06402133405208588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,128,0,1,float16,fp8,0,0.05339199801286062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,128,0,1,fp8,fp8,0,0.056320001681645714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,128,0,1,float16,float16,0,0.054133335749308266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,128,0,1,float16,fp8,0,0.05462933580080668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,128,0,1,fp8,fp8,0,0.04488533238569895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,128,0,1,float16,float16,0,0.05398400127887726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,128,0,1,fp8,fp8,0,0.04446400205294291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,128,0,1,float16,float16,0,0.05482666691144308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,128,0,1,float16,fp8,0,0.05375466744105021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,128,0,1,fp8,fp8,0,0.04660800099372864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,128,0,1,float16,float16,0,0.055231998364130654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,128,0,1,float16,fp8,0,0.05504000186920166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,128,0,1,fp8,fp8,0,0.04866133133570353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,128,0,1,fp8,fp8,0,3.505589485168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,128,0,1,float16,float16,0,3.8107519149780273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,128,0,1,float16,fp8,0,3.8944266637166343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,128,0,1,fp8,fp8,0,3.493941307067871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,128,0,1,float16,float16,0,3.8545281092325845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,128,0,1,float16,fp8,0,3.819941202799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,128,0,1,float16,float16,0,3.9692160288492837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,128,0,1,float16,float16,0,2.708832105000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,128,0,1,fp8,fp8,0,3.5498666763305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,128,0,1,float16,fp8,0,2.5938666661580405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,128,0,1,float16,fp8,0,3.952063878377279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,128,0,1,fp8,fp8,0,3.6471999486287436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,128,0,1,float16,float16,0,4.247541427612305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,128,0,1,float16,fp8,0,4.238597234090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,128,0,1,fp8,fp8,0,2.2959893544514975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,128,0,1,float16,float16,0,1.9237066904703777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,128,0,1,fp8,fp8,0,1.6771732966105144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,128,0,1,float16,fp8,0,1.8958880106608074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,128,0,1,float16,float16,0,1.9188213348388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,128,0,1,float16,fp8,0,1.912922700246175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,128,0,1,fp8,fp8,0,1.7614399592081706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,128,0,1,float16,float16,0,1.9756320317586262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,128,0,1,float16,fp8,0,1.946671962738037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,128,0,1,fp8,fp8,0,1.7838880221048992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,128,0,1,float16,float16,0,2.101568063100179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,128,0,1,float16,float16,0,1.3534399668375652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,128,0,1,float16,fp8,0,2.0795253117879233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,128,0,1,float16,fp8,0,1.290618658065796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,128,0,1,fp8,fp8,0,1.8225226402282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,128,0,1,float16,float16,0,0.9753546714782715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,128,0,1,fp8,fp8,0,1.1527573267618816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,128,0,1,float16,fp8,0,0.9636267026265463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,128,0,1,fp8,fp8,0,0.835045337677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,128,0,1,float16,float16,0,0.9690133730570475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,128,0,1,fp8,fp8,0,0.8690293629964193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,128,0,1,float16,fp8,0,0.9728853702545166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,128,0,1,float16,float16,0,0.9869653383890787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,128,0,1,float16,fp8,0,0.9939573605855306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,128,0,1,fp8,fp8,0,0.8963413238525391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,128,0,1,float16,float16,0,1.0250720183054607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,128,0,1,float16,fp8,0,1.01419202486674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,128,0,1,float16,float16,0,0.6802186965942383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,128,0,1,float16,fp8,0,0.6547146638234457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,128,0,1,float16,float16,0,0.49609601497650146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,128,0,1,fp8,fp8,0,0.9116960366566976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,128,0,1,fp8,fp8,0,0.5717759927113851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,128,0,1,float16,fp8,0,0.49273598194122314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,128,0,1,fp8,fp8,0,0.43186132113138836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,128,0,1,float16,float16,0,0.5006399949391683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,128,0,1,float16,fp8,0,0.4981120030085246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,128,0,1,fp8,fp8,0,0.43747735023498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,128,0,1,float16,float16,0,0.5058133204778036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,128,0,1,float16,fp8,0,0.5003893375396729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,128,0,1,float16,float16,0,0.5259626706441244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,128,0,1,fp8,fp8,0,0.43955198923746747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,128,0,1,float16,fp8,0,0.5194293260574341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,128,0,1,float16,float16,0,0.3527040084203084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,128,0,1,fp8,fp8,0,0.464191993077596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,128,0,1,float16,fp8,0,0.3386559883753459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,128,0,1,fp8,fp8,0,0.2919999957084656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,128,0,1,float16,fp8,0,0.2543039917945862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,128,0,1,fp8,fp8,0,0.2291520039240519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,128,0,1,float16,fp8,0,0.2519200046857198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,128,0,1,float16,float16,0,0.24606400728225708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,128,0,1,float16,fp8,0,0.2609279950459798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,128,0,1,fp8,fp8,0,0.23202133178710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,128,0,1,fp8,fp8,0,0.22758400440216064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,128,0,1,float16,float16,0,0.25170665979385376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,128,0,1,float16,float16,0,0.26421332359313965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,128,0,1,float16,float16,0,0.2685760060946147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,128,0,1,float16,fp8,0,0.2720106641451518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,128,0,1,float16,float16,0,0.1860319972038269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,128,0,1,fp8,fp8,0,0.23191465934117636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,128,0,1,float16,fp8,0,0.1819360057512919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,128,0,1,fp8,fp8,0,0.15834133823712668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,128,0,1,float16,float16,0,0.13363200426101685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,128,0,1,float16,fp8,0,0.13544533650080362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,128,0,1,fp8,fp8,0,0.11405332883199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,128,0,1,float16,float16,0,0.13844799995422363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,128,0,1,float16,fp8,0,0.1365653375784556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,128,0,1,fp8,fp8,0,0.11642666657765706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,128,0,1,float16,float16,0,0.1404906709988912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,128,0,1,float16,fp8,0,0.1413653294245402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,128,0,1,fp8,fp8,0,0.11710400382677714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,128,0,1,float16,float16,0,0.14475733041763306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,128,0,1,float16,fp8,0,0.14409599701563516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,128,0,1,fp8,fp8,0,0.1211946705977122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,128,0,1,float16,float16,0,0.11448533336321513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,128,0,1,float16,fp8,0,0.11100799838701884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,128,0,1,fp8,fp8,0,0.08545600374539693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,128,0,1,float16,float16,0,0.07673599819342296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,128,0,1,float16,fp8,0,0.07648000121116638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,128,0,1,fp8,fp8,0,0.06470933556556702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,128,0,1,float16,float16,0,0.0763679991165797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,128,0,1,float16,fp8,0,0.07798400024573009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,128,0,1,fp8,fp8,0,0.06594666838645935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,128,0,1,float16,float16,0,0.07876266539096832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,128,0,1,float16,fp8,0,0.07828799883524577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,128,0,1,fp8,fp8,0,0.06780800223350525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,128,0,1,fp8,fp8,0,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,128,0,1,float16,float16,0,0.07945600152015686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,128,0,1,float16,fp8,0,0.08054933448632558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,128,0,1,fp8,fp8,0,0.06892266869544983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,128,0,1,float16,float16,0,0.056133334835370384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,128,0,1,float16,fp8,0,0.05402133365472158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,128,0,1,float16,float16,0,0.04535999894142151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,128,0,1,float16,fp8,0,0.04480533301830292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,128,0,1,fp8,fp8,0,0.03777066618204117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,128,0,1,float16,float16,0,0.04478933413823446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,128,0,1,float16,fp8,0,0.04525866607824961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,128,0,1,fp8,fp8,0,0.038202665746212006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,128,0,1,float16,float16,0,0.0459199994802475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,128,0,1,float16,fp8,0,0.04624533156553904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,128,0,1,fp8,fp8,0,0.03890133400758108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,128,0,1,float16,float16,0,0.04637866715590159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,128,0,1,float16,fp8,0,0.046613335609436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,128,0,1,fp8,fp8,0,0.04197866717974345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,128,0,1,float16,float16,0,0.03151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,128,0,1,fp8,fp8,0,0.026698666314284008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,128,0,1,float16,fp8,0,0.032058666149775185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,128,0,1,fp8,fp8,0,0.029535998900731403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,128,0,1,float16,float16,0,0.029285334050655365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,128,0,1,float16,fp8,0,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,128,0,1,float16,float16,0,0.029658667743206024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,128,0,1,float16,fp8,0,0.02959466725587845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,128,0,1,fp8,fp8,0,0.026954665780067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,128,0,1,float16,float16,0,0.029877332349618275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,128,0,1,float16,fp8,0,0.030031998952229817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,128,0,1,fp8,fp8,0,0.027434666951497395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,128,0,1,float16,float16,0,0.030192000170548756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,128,0,1,float16,fp8,0,0.030133334298928578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,128,0,1,fp8,fp8,0,0.027658666173617046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,128,0,1,fp8,fp8,0,1.3282399972279866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,128,0,1,float16,float16,0,1.6600319544474285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,128,0,1,float16,fp8,0,1.6466933886210124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,128,0,1,float16,float16,0,1.6659626960754395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,128,0,1,fp8,fp8,0,1.3960213661193848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,128,0,1,float16,fp8,0,1.6617546081542969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,128,0,1,float16,float16,0,1.7036213874816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,128,0,1,fp8,fp8,0,1.43284273147583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,128,0,1,float16,fp8,0,1.6879253387451172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,128,0,1,float16,float16,0,1.2464426358540852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,128,0,1,float16,float16,0,1.8418879508972168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,128,0,1,fp8,fp8,0,1.4986720085144043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,128,0,1,float16,fp8,0,1.8326187133789062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,128,0,1,float16,fp8,0,1.1853439807891846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,128,0,1,float16,float16,0,0.8355200290679932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,128,0,1,fp8,fp8,0,1.0001599788665771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,128,0,1,float16,fp8,0,0.8367946942647299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,128,0,1,fp8,fp8,0,0.6633280118306478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,128,0,1,float16,float16,0,0.84279998143514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,128,0,1,fp8,fp8,0,0.6958933671315511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,128,0,1,float16,fp8,0,0.8435680071512858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,128,0,1,float16,float16,0,0.8605013688405355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,128,0,1,fp8,fp8,0,0.7213119665781657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,128,0,1,float16,fp8,0,0.8543946743011475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,128,0,1,float16,float16,0,0.8931519985198975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,128,0,1,float16,fp8,0,0.8801973660786947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,128,0,1,fp8,fp8,0,0.7462240060170492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,128,0,1,float16,float16,0,0.6270026763280233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,128,0,1,fp8,fp8,0,0.4915306568145752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,128,0,1,float16,fp8,0,0.5947519938151041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,128,0,1,float16,float16,0,0.42896533012390137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,128,0,1,float16,fp8,0,0.4266506830851237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,128,0,1,fp8,fp8,0,0.34621866544087726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,128,0,1,float16,float16,0,0.4331626494725545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,128,0,1,fp8,fp8,0,0.3479679822921753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,128,0,1,float16,fp8,0,0.4338506857554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,128,0,1,float16,float16,0,0.440778652826945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,128,0,1,float16,fp8,0,0.4385013182957967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,128,0,1,fp8,fp8,0,0.35679999987284344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,128,0,1,float16,float16,0,0.45659200350443524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,128,0,1,float16,fp8,0,0.45527998606363934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,128,0,1,fp8,fp8,0,0.37901333967844647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,128,0,1,float16,float16,0,0.3232053319613139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,128,0,1,float16,fp8,0,0.3104426662127177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,128,0,1,fp8,fp8,0,0.25512532393137616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,128,0,1,float16,float16,0,0.21278933684031168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,128,0,1,float16,fp8,0,0.21549334128697714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,128,0,1,fp8,fp8,0,0.1787573297818502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,128,0,1,float16,float16,0,0.22160534063975015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,128,0,1,float16,fp8,0,0.21902932723363241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,128,0,1,fp8,fp8,0,0.1830186645189921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,128,0,1,float16,fp8,0,0.2315466602643331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,128,0,1,float16,float16,0,0.228767991065979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,128,0,1,float16,fp8,0,0.22849067052205405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,128,0,1,fp8,fp8,0,0.1853653391202291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,128,0,1,float16,float16,0,0.2347253362337748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,128,0,1,fp8,fp8,0,0.19458667437235513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,128,0,1,float16,float16,0,0.17311465740203857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,128,0,1,float16,fp8,0,0.16637866695721945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,128,0,1,fp8,fp8,0,0.14383999506632486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,128,0,1,float16,float16,0,0.1183093289534251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,128,0,1,float16,fp8,0,0.1188213328520457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,128,0,1,fp8,fp8,0,0.10116799672444661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,128,0,1,float16,float16,0,0.11940266688664754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,128,0,1,float16,fp8,0,0.11978666981061299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,128,0,1,fp8,fp8,0,0.10382933417956035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,128,0,1,float16,float16,0,0.12350933750470479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,128,0,1,float16,fp8,0,0.12261866529782613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,128,0,1,fp8,fp8,0,0.10548800230026245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,128,0,1,float16,float16,0,0.12685867150624594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,128,0,1,fp8,fp8,0,0.08020266890525818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,128,0,1,float16,fp8,0,0.12622933586438498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,128,0,1,fp8,fp8,0,0.10794666409492493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,128,0,1,float16,float16,0,0.10652800401051839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,128,0,1,float16,fp8,0,0.10494400064150493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,128,0,1,float16,fp8,0,0.06950933237870534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,128,0,1,float16,float16,0,0.06939200063546498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,128,0,1,fp8,fp8,0,0.058261334896087646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,128,0,1,float16,fp8,0,0.06874666611353557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,128,0,1,fp8,fp8,0,0.05712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,128,0,1,float16,float16,0,0.06904000043869019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,128,0,1,float16,float16,0,0.07005333403746287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,128,0,1,float16,fp8,0,0.07041066884994507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,128,0,1,fp8,fp8,0,0.060975998640060425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,128,0,1,float16,float16,0,0.07276799778143565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,128,0,1,float16,fp8,0,0.07276266813278198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,128,0,1,float16,float16,0,0.04153066625197729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,128,0,1,fp8,fp8,0,0.06306666632493337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,128,0,1,float16,float16,0,0.05309866865475973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,128,0,1,float16,fp8,0,0.05067199965318044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,128,0,1,fp8,fp8,0,0.04783466458320618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,128,0,1,float16,fp8,0,0.04155199974775314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,128,0,1,fp8,fp8,0,0.03499199946721395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,128,0,1,float16,float16,0,0.04182933270931244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,128,0,1,float16,fp8,0,0.041573333243529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,128,0,1,fp8,fp8,0,0.03626133253177007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,128,0,1,float16,float16,0,0.04181866844495138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,128,0,1,float16,fp8,0,0.042394667863845825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,128,0,1,fp8,fp8,0,0.03716800113519033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,128,0,1,float16,float16,0,0.042597333590189614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,128,0,1,float16,fp8,0,0.043178667624791466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,128,0,1,fp8,fp8,0,0.03855466594298681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,128,0,1,float16,float16,0,0.031040000418821972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,128,0,1,float16,fp8,0,0.030400000512599945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,128,0,1,fp8,fp8,0,0.029018667836983997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,128,0,1,float16,float16,0,0.028255999088287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,128,0,1,float16,fp8,0,0.02805333336194356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,128,0,1,fp8,fp8,0,0.02643733223279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,128,0,1,float16,float16,0,0.028325334191322327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,128,0,1,fp8,fp8,0,0.026591998835404713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,128,0,1,float16,fp8,0,0.02881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,128,0,1,fp8,fp8,0,0.0266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,128,0,1,float16,float16,0,0.028789333999156952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,128,0,1,float16,fp8,0,0.0284853329261144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,128,0,1,float16,float16,0,0.02958933264017105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,128,0,1,float16,fp8,0,0.030085332691669464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,128,0,1,fp8,fp8,0,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,128,0,1,float16,float16,0,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,128,0,1,float16,fp8,0,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,128,0,1,fp8,fp8,0,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,128,0,1,float16,float16,0,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,128,0,1,float16,fp8,0,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,128,0,1,float16,float16,0,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,128,0,1,fp8,fp8,0,0.019637333850065868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,128,0,1,float16,float16,0,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,128,0,1,float16,fp8,0,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,128,0,1,fp8,fp8,0,0.020367999871571858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,128,0,1,float16,fp8,0,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,128,0,1,fp8,fp8,0,0.020693333198626835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,128,0,1,float16,float16,0,0.02149333308140437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,128,0,1,float16,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,128,0,1,fp8,fp8,0,0.02073066681623459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,128,0,1,float16,float16,0,0.7914720376332601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,128,0,1,float16,fp8,0,0.7914933363596598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,128,0,1,fp8,fp8,0,0.6649493376413981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,128,0,1,float16,float16,0,0.8006827036539713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,128,0,1,float16,fp8,0,0.8003040154774984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,128,0,1,fp8,fp8,0,0.6883573532104492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,128,0,1,float16,float16,0,0.8149706522623698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,128,0,1,float16,fp8,0,0.8096640110015869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,128,0,1,fp8,fp8,0,0.7322613398234049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,128,0,1,float16,float16,0,0.8459359804789225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,128,0,1,float16,fp8,0,0.8373973369598389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,128,0,1,float16,float16,0,0.6442026694615682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,128,0,1,fp8,fp8,0,0.7497759660085043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,128,0,1,float16,fp8,0,0.613482673962911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,128,0,1,fp8,fp8,0,0.48481067021687824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,128,0,1,float16,float16,0,0.4050453503926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,128,0,1,float16,fp8,0,0.4041546583175659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,128,0,1,float16,float16,0,0.40934399763743085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,128,0,1,fp8,fp8,0,0.34329064687093097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,128,0,1,float16,fp8,0,0.4086666504542033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,128,0,1,fp8,fp8,0,0.34407468636830646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,128,0,1,float16,float16,0,0.42046932379404706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,128,0,1,float16,fp8,0,0.4164373477300008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,128,0,1,fp8,fp8,0,0.35742934544881183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,128,0,1,float16,float16,0,0.3331200083096822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,128,0,1,float16,fp8,0,0.43187201023101807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,128,0,1,fp8,fp8,0,0.3707360029220581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,128,0,1,float16,float16,0,0.4359573523203532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,128,0,1,float16,fp8,0,0.3197919925053914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,128,0,1,float16,fp8,0,0.20291733741760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,128,0,1,fp8,fp8,0,0.25275200605392456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,128,0,1,float16,float16,0,0.20586667458216348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,128,0,1,float16,fp8,0,0.2078346610069275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,128,0,1,float16,float16,0,0.20070399840672812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,128,0,1,fp8,fp8,0,0.17827200889587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,128,0,1,float16,float16,0,0.21700799465179443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,128,0,1,fp8,fp8,0,0.1835093299547831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,128,0,1,float16,fp8,0,0.21663467089335123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,128,0,1,fp8,fp8,0,0.18517865737279257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,128,0,1,float16,float16,0,0.22524799903233847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,128,0,1,fp8,fp8,0,0.19276267290115356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,128,0,1,float16,float16,0,0.1774666706720988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,128,0,1,float16,fp8,0,0.17166399955749512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,128,0,1,fp8,fp8,0,0.10100799798965454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,128,0,1,float16,fp8,0,0.22326932350794473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,128,0,1,fp8,fp8,0,0.14016000429789224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,128,0,1,float16,fp8,0,0.1104746659596761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,128,0,1,float16,float16,0,0.11800533533096313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,128,0,1,float16,float16,0,0.11036800344785054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,128,0,1,float16,float16,0,0.1141866644223531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,128,0,1,float16,fp8,0,0.11281599601109822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,128,0,1,fp8,fp8,0,0.10186666250228882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,128,0,1,float16,fp8,0,0.11865066488583882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,128,0,1,fp8,fp8,0,0.10308800141016643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,128,0,1,float16,float16,0,0.12248532970746358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,128,0,1,float16,fp8,0,0.12177600463231404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,128,0,1,float16,float16,0,0.06422399977842967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,128,0,1,fp8,fp8,0,0.10624000430107117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,128,0,1,float16,float16,0,0.10068800052007039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,128,0,1,float16,fp8,0,0.09856533010800679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,128,0,1,fp8,fp8,0,0.0805920014778773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,128,0,1,float16,fp8,0,0.06480533381303151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,128,0,1,fp8,fp8,0,0.05680533250172933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,128,0,1,float16,float16,0,0.06462933123111725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,128,0,1,float16,fp8,0,0.06512533128261566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,128,0,1,fp8,fp8,0,0.058320000767707825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,128,0,1,float16,float16,0,0.06663466493288676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,128,0,1,float16,fp8,0,0.0669706662495931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,128,0,1,fp8,fp8,0,0.0599839985370636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,128,0,1,float16,float16,0,0.06996266543865204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,128,0,1,float16,fp8,0,0.0699946681658427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,128,0,1,fp8,fp8,0,0.06281599899133046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,128,0,1,float16,float16,0,0.050885334610939026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,128,0,1,float16,fp8,0,0.04916800061861674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,128,0,1,fp8,fp8,0,0.04683200021584829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,128,0,1,float16,float16,0,0.039647998909155525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,128,0,1,float16,fp8,0,0.03941333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,128,0,1,float16,float16,0,0.04061333338419596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,128,0,1,float16,fp8,0,0.040933333337306976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,128,0,1,fp8,fp8,0,0.0347680002450943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,128,0,1,float16,float16,0,0.039461334546407066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,128,0,1,float16,fp8,0,0.03923733284076055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,128,0,1,fp8,fp8,0,0.03479466587305069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,128,0,1,fp8,fp8,0,0.03677866607904434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,128,0,1,float16,float16,0,0.041573333243529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,128,0,1,float16,fp8,0,0.04201066493988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,128,0,1,fp8,fp8,0,0.03959999978542328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,128,0,1,float16,float16,0,0.031712000568707786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,128,0,1,float16,fp8,0,0.0312266672650973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,128,0,1,fp8,fp8,0,0.02905600021282832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,128,0,1,float16,float16,0,0.02869333326816559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,128,0,1,float16,fp8,0,0.028431999186674755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,128,0,1,fp8,fp8,0,0.026373334228992462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,128,0,1,float16,float16,0,0.028490667541821797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,128,0,1,float16,float16,0,0.02974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,128,0,1,float16,fp8,0,0.028624000648657482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,128,0,1,fp8,fp8,0,0.02607999990383784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,128,0,1,float16,float16,0,0.02908266584078471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,128,0,1,float16,fp8,0,0.029605334003766377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,128,0,1,fp8,fp8,0,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,128,0,1,float16,fp8,0,0.02940266579389572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,128,0,1,fp8,fp8,0,0.02788266787926356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,128,0,1,float16,float16,0,0.021733333667119343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,128,0,1,float16,fp8,0,0.021856000026067097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,128,0,1,fp8,fp8,0,0.020810666183630627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,128,0,1,float16,float16,0,0.019952000429232914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,128,0,1,float16,fp8,0,0.02054399996995926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,128,0,1,fp8,fp8,0,0.01978133370478948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,128,0,1,float16,float16,0,0.020053333292404812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,128,0,1,float16,fp8,0,0.020432000358899433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,128,0,1,fp8,fp8,0,0.02004266654451688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,128,0,1,float16,float16,0,0.020608000457286835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,128,0,1,float16,fp8,0,0.020794666061798733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,128,0,1,fp8,fp8,0,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,128,0,1,fp8,fp8,0,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,128,0,1,float16,float16,0,0.020448000480731327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,128,0,1,float16,fp8,0,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,128,0,1,fp8,fp8,0,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,128,0,1,fp8,fp8,0,0.020725333442290623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,128,0,1,float16,float16,0,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,128,0,1,float16,fp8,0,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,128,0,1,float16,float16,0,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,128,0,1,float16,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,128,0,1,float16,float16,0,0.016810666769742966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,128,0,1,float16,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,128,0,1,float16,fp8,0,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,128,0,1,fp8,fp8,0,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,128,0,1,float16,float16,0,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,128,0,1,float16,fp8,0,0.017749333133300144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,128,0,1,fp8,fp8,0,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,128,0,1,float16,float16,0,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,128,0,1,fp8,fp8,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,128,0,1,float16,float16,0,0.4618826707204183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,128,0,1,float16,fp8,0,0.4604693253835042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,128,0,1,fp8,fp8,0,0.48812798659006756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,128,0,1,float16,float16,0,0.4694026708602905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,128,0,1,float16,fp8,0,0.4654560089111328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,128,0,1,float16,float16,0,0.47460798422495526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,128,0,1,fp8,fp8,0,0.4909866650899251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,128,0,1,float16,fp8,0,0.4724106788635254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,128,0,1,fp8,fp8,0,0.4949653148651123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,128,0,1,float16,float16,0,0.4919360081354777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,128,0,1,float16,fp8,0,0.48842132091522217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,128,0,1,float16,float16,0,0.33486398061116535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,128,0,1,fp8,fp8,0,0.5072000026702881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,128,0,1,float16,float16,0,0.2259733279546102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,128,0,1,float16,fp8,0,0.3214293320973714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,128,0,1,fp8,fp8,0,0.3190559943517049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,128,0,1,fp8,fp8,0,0.25082133213679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,128,0,1,float16,fp8,0,0.22545599937438965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,128,0,1,float16,float16,0,0.23122666279474893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,128,0,1,float16,float16,0,0.24014399449030557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,128,0,1,float16,fp8,0,0.2295893430709839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,128,0,1,fp8,fp8,0,0.24947200218836466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,128,0,1,float16,fp8,0,0.2400053342183431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,128,0,1,fp8,fp8,0,0.2574933369954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,128,0,1,float16,float16,0,0.24644800027211508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,128,0,1,float16,fp8,0,0.2456159989039103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,128,0,1,fp8,fp8,0,0.26073600848515827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,128,0,1,float16,float16,0,0.17642132441202799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,128,0,1,float16,fp8,0,0.1697439948717753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,128,0,1,fp8,fp8,0,0.16904000441233316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,128,0,1,float16,float16,0,0.12081600228945415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,128,0,1,float16,fp8,0,0.11994133392969768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,128,0,1,fp8,fp8,0,0.13517333070437113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,128,0,1,float16,float16,0,0.1232373317082723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,128,0,1,float16,fp8,0,0.12247999509175618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,128,0,1,fp8,fp8,0,0.13826666275660196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,128,0,1,fp8,fp8,0,0.13550399740537009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,128,0,1,float16,float16,0,0.1271519958972931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,128,0,1,float16,fp8,0,0.1268160045146942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,128,0,1,float16,float16,0,0.13158399860064188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,128,0,1,float16,fp8,0,0.12985600034395853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,128,0,1,fp8,fp8,0,0.14034133156140646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,128,0,1,float16,fp8,0,0.06815466781457265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,128,0,1,float16,float16,0,0.09678399562835693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,128,0,1,fp8,fp8,0,0.07534933090209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,128,0,1,float16,fp8,0,0.09477866689364116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,128,0,1,fp8,fp8,0,0.09463999668757121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,128,0,1,float16,float16,0,0.06798399984836578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,128,0,1,float16,float16,0,0.06828799843788147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,128,0,1,float16,fp8,0,0.06871999800205231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,128,0,1,fp8,fp8,0,0.07635733485221863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,128,0,1,float16,float16,0,0.0698773314555486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,128,0,1,float16,fp8,0,0.06941866874694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,128,0,1,fp8,fp8,0,0.07843733330567677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,128,0,1,float16,float16,0,0.0726506660381953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,128,0,1,float16,fp8,0,0.07241599758466084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,128,0,1,fp8,fp8,0,0.0798826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,128,0,1,float16,float16,0,0.05039466420809428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,128,0,1,float16,fp8,0,0.04971200227737427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,128,0,1,fp8,fp8,0,0.05342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,128,0,1,float16,float16,0,0.039450667798519135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,128,0,1,float16,fp8,0,0.0394400010506312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,128,0,1,fp8,fp8,0,0.04267199834187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,128,0,1,float16,float16,0,0.039461334546407066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,128,0,1,float16,fp8,0,0.039408000806967415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,128,0,1,fp8,fp8,0,0.04258666435877482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,128,0,1,float16,float16,0,0.040693332751592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,128,0,1,float16,fp8,0,0.04058666775623957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,128,0,1,fp8,fp8,0,0.0441599984963735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,128,0,1,float16,float16,0,0.0405973345041275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,128,0,1,float16,fp8,0,0.04127466678619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,128,0,1,float16,fp8,0,0.028037334481875103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,128,0,1,fp8,fp8,0,0.04640533526738485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,128,0,1,float16,float16,0,0.0305173322558403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,128,0,1,float16,fp8,0,0.030245333909988403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,128,0,1,fp8,fp8,0,0.03299733251333237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,128,0,1,float16,float16,0,0.028175999720891316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,128,0,1,fp8,fp8,0,0.02961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,128,0,1,float16,float16,0,0.028229333460330963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,128,0,1,float16,fp8,0,0.028442665934562683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,128,0,1,fp8,fp8,0,0.02974933385848999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,128,0,1,float16,float16,0,0.02810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,128,0,1,float16,fp8,0,0.02861333390076955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,128,0,1,fp8,fp8,0,0.030720000465710957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,128,0,1,float16,float16,0,0.028704000016053517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,128,0,1,float16,fp8,0,0.02905600021282832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,128,0,1,fp8,fp8,0,0.03178133318821589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,128,0,1,float16,float16,0,0.023397333920001984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,128,0,1,float16,fp8,0,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,128,0,1,fp8,fp8,0,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,128,0,1,float16,float16,0,0.02216000109910965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,128,0,1,float16,fp8,0,0.02271466702222824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,128,0,1,fp8,fp8,0,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,128,0,1,float16,float16,0,0.022410665949185688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,128,0,1,float16,fp8,0,0.022250667214393616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,128,0,1,fp8,fp8,0,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,128,0,1,float16,float16,0,0.02182399978240331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,128,0,1,float16,fp8,0,0.02231466770172119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,128,0,1,fp8,fp8,0,0.02365333338578542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,128,0,1,float16,float16,0,0.022495999932289124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,128,0,1,float16,fp8,0,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,128,0,1,fp8,fp8,0,0.024432001014550526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,128,0,1,float16,float16,0,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,128,0,1,float16,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,128,0,1,fp8,fp8,0,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,128,0,1,float16,float16,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,128,0,1,float16,fp8,0,0.016186666985352833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,128,0,1,fp8,fp8,0,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,128,0,1,float16,float16,0,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,128,0,1,float16,fp8,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,128,0,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,128,0,1,float16,float16,0,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,128,0,1,float16,fp8,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,128,0,1,fp8,fp8,0,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,128,0,1,float16,float16,0,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,128,0,1,float16,fp8,0,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,128,0,1,fp8,fp8,0,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,128,0,1,float16,float16,0,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,128,0,1,float16,fp8,0,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,128,0,1,fp8,fp8,0,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,128,0,1,float16,float16,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,128,0,1,float16,fp8,0,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,128,0,1,fp8,fp8,0,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,128,0,1,float16,float16,0,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,128,0,1,float16,fp8,0,0.015642666568358738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,128,0,1,fp8,fp8,0,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,128,0,1,float16,float16,0,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,128,0,1,float16,fp8,0,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,128,0,1,fp8,fp8,0,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,128,0,1,float16,float16,0,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,128,0,1,float16,fp8,0,0.01597333326935768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,128,0,1,fp8,fp8,0,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,128,0,1,float16,fp8,0,0.3025439977645874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,128,0,1,float16,float16,0,0.3019413352012634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,128,0,1,float16,float16,0,0.3088159958521525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,128,0,1,fp8,fp8,0,0.3858826557795207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,128,0,1,float16,fp8,0,0.3075679937998454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,128,0,1,fp8,fp8,0,0.3879306713740031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,128,0,1,float16,float16,0,0.3186560074488322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,128,0,1,fp8,fp8,0,0.3912320137023926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,128,0,1,float16,fp8,0,0.31780799229939777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,128,0,1,float16,float16,0,0.3288693428039551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,128,0,1,float16,fp8,0,0.3276960055033366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,128,0,1,float16,float16,0,0.21789334217707315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,128,0,1,fp8,fp8,0,0.3957013289133708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,128,0,1,float16,fp8,0,0.21107733249664307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,128,0,1,fp8,fp8,0,0.23577600717544556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,128,0,1,float16,float16,0,0.15901333093643188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,128,0,1,float16,fp8,0,0.1588533322016398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,128,0,1,fp8,fp8,0,0.20363734165827432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,128,0,1,float16,float16,0,0.16173332929611206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,128,0,1,float16,fp8,0,0.16113600134849548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,128,0,1,fp8,fp8,0,0.20337599515914917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,128,0,1,float16,fp8,0,0.17112533251444498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,128,0,1,float16,float16,0,0.16703466574350992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,128,0,1,float16,fp8,0,0.16664533813794455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,128,0,1,fp8,fp8,0,0.20538665850957236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,128,0,1,float16,float16,0,0.17209599415461221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,128,0,1,fp8,fp8,0,0.20878932873408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,128,0,1,float16,float16,0,0.11589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,128,0,1,float16,fp8,0,0.11211199561754863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,128,0,1,fp8,fp8,0,0.1308693289756775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,128,0,1,float16,float16,0,0.08616532882054646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,128,0,1,float16,fp8,0,0.08684800068537395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,128,0,1,fp8,fp8,0,0.10916800300280254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,128,0,1,float16,float16,0,0.0869653324286143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,128,0,1,float16,fp8,0,0.08662399649620056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,128,0,1,fp8,fp8,0,0.10984533031781514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,128,0,1,float16,float16,0,0.08843732873598735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,128,0,1,float16,fp8,0,0.0883679986000061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,128,0,1,fp8,fp8,0,0.11181867122650146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,128,0,1,float16,float16,0,0.09268266956011455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,128,0,1,float16,fp8,0,0.09227733810742696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,128,0,1,fp8,fp8,0,0.11334932843844096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,128,0,1,float16,float16,0,0.06005333364009857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,128,0,1,float16,fp8,0,0.05823466678460439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,128,0,1,fp8,fp8,0,0.07124799986680348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,128,0,1,fp8,fp8,0,0.0618399977684021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,128,0,1,float16,float16,0,0.049072002371152244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,128,0,1,float16,float16,0,0.048138668139775596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,128,0,1,float16,fp8,0,0.04816000163555145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,128,0,1,fp8,fp8,0,0.06028266747792562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,128,0,1,float16,float16,0,0.04809600114822388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,128,0,1,float16,fp8,0,0.04801600178082784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,128,0,1,float16,fp8,0,0.04865066707134247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,128,0,1,fp8,fp8,0,0.06344533463319142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,128,0,1,float16,float16,0,0.030565333863099415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,128,0,1,float16,float16,0,0.049866666396458946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,128,0,1,float16,fp8,0,0.051226665576299034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,128,0,1,fp8,fp8,0,0.06423999865849812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,128,0,1,float16,float16,0,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,128,0,1,float16,fp8,0,0.032858667274316154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,128,0,1,fp8,fp8,0,0.03697066754102707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,128,0,1,float16,float16,0,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,128,0,1,fp8,fp8,0,0.0405173326532046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,128,0,1,float16,fp8,0,0.030826665461063385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,128,0,1,fp8,fp8,0,0.03686933219432831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,128,0,1,float16,float16,0,0.0303413321574529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,128,0,1,float16,fp8,0,0.030960001051425934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,128,0,1,float16,fp8,0,0.031008000175158184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,128,0,1,fp8,fp8,0,0.037776000797748566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,128,0,1,float16,float16,0,0.030762667457262676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,128,0,1,float16,fp8,0,0.03176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,128,0,1,fp8,fp8,0,0.03956266740957896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,128,0,1,float16,float16,0,0.024197332561016083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,128,0,1,float16,fp8,0,0.024112001061439514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,128,0,1,fp8,fp8,0,0.02940266579389572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,128,0,1,float16,float16,0,0.022298666338125866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,128,0,1,float16,fp8,0,0.022341333329677582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,128,0,1,fp8,fp8,0,0.027072000006834667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,128,0,1,float16,float16,0,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,128,0,1,float16,fp8,0,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,128,0,1,fp8,fp8,0,0.026554666459560394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,128,0,1,float16,float16,0,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,128,0,1,float16,fp8,0,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,128,0,1,fp8,fp8,0,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,128,0,1,float16,fp8,0,0.01985599969824155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,128,0,1,float16,fp8,0,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,128,0,1,fp8,fp8,0,0.028618666032950085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,128,0,1,float16,float16,0,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,128,0,1,fp8,fp8,0,0.023221333821614582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,128,0,1,float16,float16,0,0.01815466706951459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,128,0,1,float16,fp8,0,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,128,0,1,fp8,fp8,0,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,128,0,1,float16,float16,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,128,0,1,float16,fp8,0,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,128,0,1,fp8,fp8,0,0.020714666694402695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,128,0,1,float16,float16,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,128,0,1,float16,fp8,0,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,128,0,1,fp8,fp8,0,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,128,0,1,float16,float16,0,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,128,0,1,fp8,fp8,0,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,128,0,1,float16,fp8,0,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,128,0,1,fp8,fp8,0,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,128,0,1,float16,float16,0,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,128,0,1,float16,fp8,0,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,128,0,1,float16,float16,0,0.015583999454975128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,128,0,1,float16,fp8,0,0.01573333392540614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,128,0,1,fp8,fp8,0,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,128,0,1,float16,float16,0,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,128,0,1,float16,fp8,0,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,128,0,1,fp8,fp8,0,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,128,0,1,float16,float16,0,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,128,0,1,float16,fp8,0,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,128,0,1,fp8,fp8,0,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,128,0,1,float16,float16,0,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,128,0,1,float16,fp8,0,0.015728000551462173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,128,0,1,fp8,fp8,0,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,128,0,1,float16,float16,0,0.014618666221698126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,128,0,1,float16,fp8,0,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,128,0,1,fp8,fp8,0,0.01681600014368693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,128,0,1,float16,float16,0,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,128,0,1,float16,fp8,0,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,128,0,1,float16,float16,0,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,128,0,1,fp8,fp8,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,128,0,1,float16,float16,0,0.0143306665122509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,128,0,1,float16,fp8,0,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,128,0,1,fp8,fp8,0,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,128,0,1,float16,fp8,0,0.014618666221698126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,128,0,1,fp8,fp8,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,128,0,1,float16,float16,0,0.014469332993030548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,128,0,1,float16,fp8,0,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,128,0,1,fp8,fp8,0,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,128,0,1,float16,float16,0,0.24581333001454672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,128,0,1,float16,fp8,0,0.24653865893681845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,128,0,1,fp8,fp8,0,0.34010668595631915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,128,0,1,float16,float16,0,0.24985599517822266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,128,0,1,float16,float16,0,0.2555466691652934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,128,0,1,fp8,fp8,0,0.34274133046468097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,128,0,1,float16,fp8,0,0.2526026765505473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,128,0,1,float16,fp8,0,0.2566506663958232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,128,0,1,fp8,fp8,0,0.343610684076945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,128,0,1,float16,float16,0,0.26310400168100995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,128,0,1,float16,fp8,0,0.2606293360392253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,128,0,1,float16,float16,0,0.1590773363908132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,128,0,1,fp8,fp8,0,0.3455253442128499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,128,0,1,float16,float16,0,0.1288053294022878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,128,0,1,float16,fp8,0,0.15944533546765646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,128,0,1,float16,fp8,0,0.1291306714216868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,128,0,1,fp8,fp8,0,0.1943626602490743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,128,0,1,fp8,fp8,0,0.17811733484268188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,128,0,1,float16,fp8,0,0.13327999909718832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,128,0,1,float16,float16,0,0.1297813355922699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,128,0,1,float16,fp8,0,0.129530668258667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,128,0,1,fp8,fp8,0,0.17916800578435263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,128,0,1,float16,float16,0,0.13127467036247253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,128,0,1,fp8,fp8,0,0.18179200092951456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,128,0,1,fp8,fp8,0,0.10602133472760518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,128,0,1,float16,float16,0,0.13682666420936584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,128,0,1,float16,fp8,0,0.13491732875506082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,128,0,1,fp8,fp8,0,0.18395733833312988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,128,0,1,float16,float16,0,0.08205333352088928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,128,0,1,float16,fp8,0,0.07963733375072479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,128,0,1,float16,float16,0,0.07028799752394359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,128,0,1,float16,fp8,0,0.07041066884994507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,128,0,1,fp8,fp8,0,0.09529067079226176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,128,0,1,float16,float16,0,0.07231466472148895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,128,0,1,float16,fp8,0,0.07049066821734111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,128,0,1,fp8,fp8,0,0.09546666344006856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,128,0,1,float16,float16,0,0.07116800049940745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,128,0,1,float16,fp8,0,0.07144533097743988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,128,0,1,fp8,fp8,0,0.09844799836476643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,128,0,1,float16,float16,0,0.07187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,128,0,1,float16,fp8,0,0.07196799914042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,128,0,1,fp8,fp8,0,0.09963732957839966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,128,0,1,float16,float16,0,0.045066664616266884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,128,0,1,float16,fp8,0,0.04381333291530609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,128,0,1,float16,float16,0,0.04249600072701772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,128,0,1,fp8,fp8,0,0.057631999254226685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,128,0,1,float16,float16,0,0.04227200150489807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,128,0,1,float16,fp8,0,0.04212800165017446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,128,0,1,fp8,fp8,0,0.05494933327039083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,128,0,1,float16,fp8,0,0.042117332418759666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,128,0,1,float16,float16,0,0.042352000872294106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,128,0,1,fp8,fp8,0,0.05575466652711233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,128,0,1,float16,float16,0,0.04205866654713949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,128,0,1,float16,fp8,0,0.04282666742801666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,128,0,1,fp8,fp8,0,0.05541866521040598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,128,0,1,float16,fp8,0,0.043141335248947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,128,0,1,float16,float16,0,0.02720533311367035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,128,0,1,fp8,fp8,0,0.05648000041643778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,128,0,1,float16,float16,0,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,128,0,1,float16,fp8,0,0.029802667597929638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,128,0,1,fp8,fp8,0,0.03608533243338267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,128,0,1,float16,fp8,0,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,128,0,1,fp8,fp8,0,0.03490666548411051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,128,0,1,float16,float16,0,0.027914665639400482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,128,0,1,float16,float16,0,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,128,0,1,float16,fp8,0,0.028394666810830433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,128,0,1,fp8,fp8,0,0.03467733412981033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,128,0,1,float16,fp8,0,0.028197333216667175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,128,0,1,fp8,fp8,0,0.034847999612490334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,128,0,1,float16,float16,0,0.027888000011444092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,128,0,1,float16,fp8,0,0.02809600035349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,128,0,1,fp8,fp8,0,0.03601066768169403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,128,0,1,float16,float16,0,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,128,0,1,float16,fp8,0,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,128,0,1,fp8,fp8,0,0.02701866626739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,128,0,1,float16,float16,0,0.019930666933457058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,128,0,1,float16,fp8,0,0.020666666328907013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,128,0,1,float16,float16,0,0.02051199972629547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,128,0,1,fp8,fp8,0,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,128,0,1,fp8,fp8,0,0.026026666164398193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,128,0,1,float16,float16,0,0.02062400057911873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,128,0,1,float16,fp8,0,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,128,0,1,fp8,fp8,0,0.025461333493391674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,128,0,1,float16,fp8,0,0.020538666596015293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,128,0,1,float16,float16,0,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,128,0,1,float16,float16,0,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,128,0,1,float16,fp8,0,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,128,0,1,fp8,fp8,0,0.02611200014750163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,128,0,1,float16,float16,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,128,0,1,float16,fp8,0,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,128,0,1,fp8,fp8,0,0.021546666820844013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,128,0,1,float16,fp8,0,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,128,0,1,fp8,fp8,0,0.020293333878119785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,128,0,1,float16,float16,0,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,128,0,1,float16,fp8,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,128,0,1,fp8,fp8,0,0.02065066620707512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,128,0,1,float16,float16,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,128,0,1,float16,fp8,0,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,128,0,1,fp8,fp8,0,0.020517333100239437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,128,0,1,float16,float16,0,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,128,0,1,float16,fp8,0,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,128,0,1,fp8,fp8,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,128,0,1,float16,float16,0,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,128,0,1,float16,fp8,0,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,128,0,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,128,0,1,float16,float16,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,128,0,1,float16,fp8,0,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,128,0,1,fp8,fp8,0,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,128,0,1,float16,float16,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,128,0,1,float16,fp8,0,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,128,0,1,fp8,fp8,0,0.016528000434239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,128,0,1,float16,float16,0,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,128,0,1,float16,float16,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,128,0,1,float16,fp8,0,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,128,0,1,fp8,fp8,0,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,128,0,1,float16,float16,0,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,128,0,1,float16,fp8,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,128,0,1,fp8,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,128,0,1,float16,fp8,0,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,128,0,1,fp8,fp8,0,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,128,0,1,float16,float16,0,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,128,0,1,float16,fp8,0,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,128,0,1,fp8,fp8,0,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,128,0,1,float16,float16,0,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,128,0,1,float16,fp8,0,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,128,0,1,fp8,fp8,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,128,0,1,float16,float16,0,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,128,0,1,float16,fp8,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,128,0,1,fp8,fp8,0,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,128,0,1,float16,float16,0,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,128,0,1,float16,fp8,0,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,128,0,1,fp8,fp8,0,0.01600533351302147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,128,0,1,fp8,fp8,0,22.083969116210938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,128,0,1,fp8,fp8,0,21.82733408610026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,128,0,1,float16,float16,0,33.54504648844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,128,0,1,float16,fp8,0,34.899803161621094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,128,0,1,float16,fp8,0,33.817125956217446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,128,0,1,float16,float16,0,34.95020294189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,128,0,1,float16,fp8,0,35.23357391357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,128,0,1,float16,float16,0,37.055765787760414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,128,0,1,fp8,fp8,0,22.564437866210938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,128,0,1,fp8,fp8,0,11.608565012613932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,128,0,1,float16,float16,0,17.00159962972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,128,0,1,float16,fp8,0,17.03226089477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,128,0,1,fp8,fp8,0,22.078196207682293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,128,0,1,float16,float16,0,33.34348805745443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,128,0,1,fp8,fp8,0,11.251925150553385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,128,0,1,float16,float16,0,16.392805735270183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,128,0,1,float16,fp8,0,16.6419677734375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,128,0,1,float16,fp8,0,33.17956288655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,128,0,1,fp8,fp8,0,11.911930084228516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,128,0,1,float16,float16,0,17.150506337483723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,128,0,1,float16,fp8,0,17.797653198242188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,128,0,1,fp8,fp8,0,11.13800048828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,128,0,1,float16,float16,0,18.116682688395183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,128,0,1,float16,fp8,0,18.085642496744793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,128,0,1,fp8,fp8,0,7.5376536051432295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,128,0,1,float16,float16,0,9.15339724222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,128,0,1,float16,fp8,0,8.869941075642904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,128,0,1,fp8,fp8,0,11.386730194091797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,128,0,1,float16,float16,0,17.53641128540039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,128,0,1,float16,float16,0,8.339690526326498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,128,0,1,fp8,fp8,0,5.732922871907552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,128,0,1,float16,fp8,0,17.216106414794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,128,0,1,fp8,fp8,0,5.622016270955403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,128,0,1,float16,fp8,0,8.716517130533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,128,0,1,float16,float16,0,8.56495475769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,128,0,1,float16,fp8,0,8.928346633911133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,128,0,1,float16,float16,0,8.545904159545898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,128,0,1,fp8,fp8,0,5.615360260009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,128,0,1,float16,fp8,0,8.600645065307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,128,0,1,float16,float16,0,4.434746742248535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,128,0,1,fp8,fp8,0,6.042837142944336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,128,0,1,float16,fp8,0,4.532554626464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,128,0,1,float16,float16,0,8.391141255696615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,128,0,1,fp8,fp8,0,2.9939467112223306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,128,0,1,float16,fp8,0,8.61837895711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,128,0,1,float16,float16,0,4.272666613260905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,128,0,1,fp8,fp8,0,2.912266731262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,128,0,1,float16,fp8,0,4.475258509318034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,128,0,1,fp8,fp8,0,2.8529065450032554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,128,0,1,float16,float16,0,4.473498662312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,128,0,1,float16,fp8,0,4.312543869018555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,128,0,1,fp8,fp8,0,2.8929599126180015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,128,0,1,float16,float16,0,4.346991856892903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,128,0,1,float16,fp8,0,4.42961057027181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,128,0,1,float16,float16,0,4.540154774983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,128,0,1,fp8,fp8,0,3.061882654825846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,128,0,1,float16,fp8,0,4.563397407531738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,128,0,1,fp8,fp8,0,13.693061828613281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,128,0,1,fp8,fp8,0,13.576133728027344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,128,0,1,float16,fp8,0,19.517578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,128,0,1,float16,float16,0,20.263071695963543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,128,0,1,float16,float16,0,19.998736063639324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,128,0,1,float16,fp8,0,19.276554107666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,128,0,1,float16,float16,0,20.166522979736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,128,0,1,float16,fp8,0,19.25427754720052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,128,0,1,fp8,fp8,0,13.297274271647135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,128,0,1,fp8,fp8,0,6.719743728637695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,128,0,1,float16,float16,0,10.205509185791016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,128,0,1,float16,fp8,0,10.057098388671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,128,0,1,float16,float16,0,9.876730600992838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,128,0,1,fp8,fp8,0,13.665269215901693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,128,0,1,float16,float16,0,19.749370574951172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,128,0,1,fp8,fp8,0,6.702986399332683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,128,0,1,float16,fp8,0,9.708714803059896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,128,0,1,float16,fp8,0,19.785546620686848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,128,0,1,fp8,fp8,0,6.794986724853516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,128,0,1,float16,float16,0,9.952234903971354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,128,0,1,float16,fp8,0,10.080389022827148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,128,0,1,fp8,fp8,0,6.693445205688477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,128,0,1,float16,float16,0,10.08953603108724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,128,0,1,float16,fp8,0,10.008879979451498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,128,0,1,float16,float16,0,5.212224006652832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,128,0,1,fp8,fp8,0,3.5179465611775718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,128,0,1,float16,float16,0,9.939648310343424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,128,0,1,float16,fp8,0,5.328010559082031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,128,0,1,fp8,fp8,0,6.964501063028972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,128,0,1,float16,float16,0,5.184117317199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,128,0,1,float16,fp8,0,10.12119992574056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,128,0,1,fp8,fp8,0,3.448042551676432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,128,0,1,float16,fp8,0,5.015551884969075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,128,0,1,fp8,fp8,0,3.3504212697347007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,128,0,1,float16,float16,0,5.176368077596028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,128,0,1,float16,fp8,0,5.303925196329753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,128,0,1,float16,float16,0,4.993253389994304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,128,0,1,fp8,fp8,0,3.388970692952474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,128,0,1,float16,fp8,0,5.26254399617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,128,0,1,fp8,fp8,0,3.5797227223714194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,128,0,1,fp8,fp8,0,1.7813547452290852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,128,0,1,float16,float16,0,2.730208079020182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,128,0,1,float16,fp8,0,2.7131786346435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,128,0,1,float16,float16,0,5.064789454142253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,128,0,1,float16,fp8,0,5.105381329854329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,128,0,1,float16,float16,0,2.6050186157226562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,128,0,1,fp8,fp8,0,1.7501600583394368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,128,0,1,float16,fp8,0,2.639354705810547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,128,0,1,fp8,fp8,0,1.799898624420166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,128,0,1,float16,float16,0,2.4307093620300293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,128,0,1,float16,fp8,0,2.667189280192057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,128,0,1,fp8,fp8,0,1.7432373364766438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,128,0,1,float16,float16,0,2.553823947906494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,128,0,1,float16,fp8,0,2.4839733441670737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,128,0,1,float16,float16,0,2.474005381266276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,128,0,1,fp8,fp8,0,1.7596747080485027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,128,0,1,float16,fp8,0,2.7930078506469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,128,0,1,fp8,fp8,0,9.531338373819986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,128,0,1,fp8,fp8,0,9.412810643513998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,128,0,1,float16,float16,0,13.821194966634115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,128,0,1,float16,fp8,0,13.932027180989584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,128,0,1,float16,float16,0,13.906997680664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,128,0,1,float16,fp8,0,13.780821482340494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,128,0,1,float16,float16,0,13.720106760660807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,128,0,1,fp8,fp8,0,9.33357302347819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,128,0,1,fp8,fp8,0,4.97054926554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,128,0,1,float16,float16,0,7.3354136149088545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,128,0,1,float16,fp8,0,7.548901240030925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,128,0,1,fp8,fp8,0,9.894997278849283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,128,0,1,float16,fp8,0,14.278549194335938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,128,0,1,float16,float16,0,13.954784393310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,128,0,1,float16,float16,0,7.179205576578776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,128,0,1,fp8,fp8,0,4.815557479858398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,128,0,1,float16,fp8,0,14.182623545328775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,128,0,1,float16,fp8,0,7.168282826741536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,128,0,1,fp8,fp8,0,4.732661247253418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,128,0,1,float16,float16,0,7.131711959838867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,128,0,1,float16,fp8,0,7.2258561452229815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,128,0,1,fp8,fp8,0,4.872373263041179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,128,0,1,float16,float16,0,7.112261454264323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,128,0,1,float16,float16,0,3.636373202006022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,128,0,1,float16,fp8,0,7.074933369954427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,128,0,1,fp8,fp8,0,4.825178782145183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,128,0,1,float16,fp8,0,3.6387198766072593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,128,0,1,float16,float16,0,7.163973490397136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,128,0,1,fp8,fp8,0,2.5137759844462075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,128,0,1,float16,fp8,0,7.313285191853841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,128,0,1,float16,float16,0,3.5613600413004556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,128,0,1,fp8,fp8,0,2.4009706179300943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,128,0,1,float16,fp8,0,3.5247891743977866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,128,0,1,fp8,fp8,0,2.39682674407959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,128,0,1,float16,float16,0,3.423184076944987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,128,0,1,float16,fp8,0,3.575733184814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,128,0,1,fp8,fp8,0,2.3477120399475098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,128,0,1,float16,float16,0,3.555466651916504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,128,0,1,float16,fp8,0,3.4814933141072593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,128,0,1,fp8,fp8,0,1.2618133227030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,128,0,1,float16,float16,0,1.86735995610555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,128,0,1,fp8,fp8,0,2.3936266899108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,128,0,1,float16,float16,0,3.768437385559082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,128,0,1,float16,fp8,0,1.8440052668253581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,128,0,1,float16,float16,0,1.7642347017923992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,128,0,1,float16,fp8,0,3.5552854537963867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,128,0,1,float16,fp8,0,1.8945226669311523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,128,0,1,fp8,fp8,0,1.2142186959584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,128,0,1,float16,float16,0,1.8114933967590332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,128,0,1,fp8,fp8,0,1.351151943206787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,128,0,1,float16,fp8,0,1.8022027015686035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,128,0,1,fp8,fp8,0,1.2228319644927979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,128,0,1,float16,float16,0,1.7216960589090984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,128,0,1,float16,fp8,0,1.7634933789571126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,128,0,1,float16,float16,0,1.8177706400553386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,128,0,1,float16,fp8,0,1.7649386723836262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,128,0,1,fp8,fp8,0,1.258944034576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,128,0,1,fp8,fp8,0,13.016639709472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,128,0,1,fp8,fp8,0,12.368272145589193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,128,0,1,float16,fp8,0,18.358357747395832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,128,0,1,float16,float16,0,18.51702372233073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,128,0,1,float16,float16,0,18.66569646199544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,128,0,1,float16,fp8,0,19.446715037027996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,128,0,1,float16,float16,0,18.457599639892578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,128,0,1,float16,fp8,0,19.07046890258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,128,0,1,fp8,fp8,0,12.769354502360025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,128,0,1,fp8,fp8,0,6.86842664082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,128,0,1,float16,float16,0,9.55283228556315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,128,0,1,float16,fp8,0,9.698554356892904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,128,0,1,float16,float16,0,9.159727732340494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,128,0,1,fp8,fp8,0,12.839173634847006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,128,0,1,float16,float16,0,18.907568613688152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,128,0,1,fp8,fp8,0,6.341023763020833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,128,0,1,float16,fp8,0,9.212730407714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,128,0,1,fp8,fp8,0,6.494218826293945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,128,0,1,float16,fp8,0,18.523887634277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,128,0,1,float16,float16,0,8.958773295084635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,128,0,1,float16,fp8,0,9.170842488606771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,128,0,1,fp8,fp8,0,6.291903813680013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,128,0,1,float16,float16,0,9.712709426879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,128,0,1,float16,fp8,0,9.029642740885416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,128,0,1,float16,float16,0,4.683109283447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,128,0,1,float16,fp8,0,4.801247914632161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,128,0,1,fp8,fp8,0,3.395493189493815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,128,0,1,fp8,fp8,0,6.301829020182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,128,0,1,float16,float16,0,9.320965449015299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,128,0,1,float16,fp8,0,9.249130884806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,128,0,1,float16,float16,0,4.6302134195963545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,128,0,1,float16,fp8,0,4.602997461954753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,128,0,1,fp8,fp8,0,3.089829444885254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,128,0,1,fp8,fp8,0,3.0968052546183267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,128,0,1,float16,float16,0,4.675722757975261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,128,0,1,float16,fp8,0,4.705984115600586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,128,0,1,float16,float16,0,4.616511980692546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,128,0,1,fp8,fp8,0,3.1293811798095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,128,0,1,float16,fp8,0,4.5867360432942705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,128,0,1,float16,float16,0,2.3632532755533853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,128,0,1,float16,float16,0,4.599024136861165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,128,0,1,float16,fp8,0,4.717946688334147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,128,0,1,fp8,fp8,0,3.31168524424235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,128,0,1,fp8,fp8,0,1.694485346476237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,128,0,1,float16,fp8,0,2.4053173065185547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,128,0,1,float16,float16,0,2.3639520009358725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,128,0,1,fp8,fp8,0,1.5446666081746419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,128,0,1,float16,fp8,0,2.3123146692911782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,128,0,1,fp8,fp8,0,1.5674187342325847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,128,0,1,float16,float16,0,2.3103307088216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,128,0,1,float16,fp8,0,2.3207680384318032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,128,0,1,float16,float16,0,2.3057066599527993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,128,0,1,fp8,fp8,0,1.5542826652526855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,128,0,1,float16,fp8,0,2.3423306147257485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,128,0,1,float16,float16,0,1.2170560359954834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,128,0,1,float16,float16,0,2.389029343922933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,128,0,1,fp8,fp8,0,1.6827467282613118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,128,0,1,float16,fp8,0,1.2294560273488362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,128,0,1,float16,fp8,0,2.371280034383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,128,0,1,fp8,fp8,0,0.887114683787028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,128,0,1,float16,float16,0,1.2098026275634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,128,0,1,float16,fp8,0,1.1804746786753337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,128,0,1,fp8,fp8,0,0.8682080109914144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,128,0,1,float16,float16,0,1.1905759970347087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,128,0,1,fp8,fp8,0,0.9112160205841064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,128,0,1,float16,fp8,0,1.1729706923166912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,128,0,1,float16,float16,0,1.1594879627227783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,128,0,1,float16,fp8,0,1.1707253456115723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,128,0,1,fp8,fp8,0,0.8825120131174723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,128,0,1,float16,float16,0,1.16267196337382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,128,0,1,fp8,fp8,0,0.8875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,128,0,1,float16,fp8,0,1.1694080034891765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,128,0,1,fp8,fp8,0,7.60148811340332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,128,0,1,fp8,fp8,0,7.682645161946614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,128,0,1,float16,float16,0,10.479354858398438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,128,0,1,float16,fp8,0,10.56943448384603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,128,0,1,float16,fp8,0,10.555685043334961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,128,0,1,float16,float16,0,10.960219065348307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,128,0,1,float16,float16,0,11.029642740885416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,128,0,1,float16,fp8,0,10.65499750773112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,128,0,1,fp8,fp8,0,7.512501398722331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,128,0,1,fp8,fp8,0,4.1702454884847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,128,0,1,float16,float16,0,5.604378382364909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,128,0,1,float16,fp8,0,5.6274668375651045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,128,0,1,float16,float16,0,5.384192148844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,128,0,1,fp8,fp8,0,7.655194600423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,128,0,1,float16,float16,0,10.69925308227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,128,0,1,fp8,fp8,0,3.8508052825927734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,128,0,1,float16,fp8,0,5.418058395385742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,128,0,1,float16,fp8,0,10.668949127197266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,128,0,1,fp8,fp8,0,3.7040160497029624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,128,0,1,float16,float16,0,5.422117233276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,128,0,1,float16,fp8,0,5.376714706420898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,128,0,1,float16,float16,0,5.44920031229655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,128,0,1,fp8,fp8,0,3.84661865234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,128,0,1,float16,fp8,0,5.717007954915364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,128,0,1,fp8,fp8,0,4.049578666687012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,128,0,1,float16,float16,0,2.919919967651367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,128,0,1,float16,fp8,0,2.8700265884399414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,128,0,1,float16,float16,0,5.519711812337239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,128,0,1,fp8,fp8,0,2.010741392771403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,128,0,1,float16,fp8,0,5.321226755777995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,128,0,1,float16,float16,0,2.745941480000814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,128,0,1,float16,fp8,0,2.5446240107218423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,128,0,1,fp8,fp8,0,1.8498400052388508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,128,0,1,fp8,fp8,0,1.8662400245666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,128,0,1,float16,float16,0,2.7249228159586587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,128,0,1,float16,fp8,0,2.662245273590088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,128,0,1,float16,float16,0,2.7423413594563804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,128,0,1,float16,fp8,0,2.640890598297119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,128,0,1,fp8,fp8,0,1.884335994720459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,128,0,1,float16,float16,0,2.8568747838338218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,128,0,1,float16,float16,0,1.431173324584961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,128,0,1,fp8,fp8,0,1.9653813044230144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,128,0,1,fp8,fp8,0,1.029136021931966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,128,0,1,float16,fp8,0,1.3973706563313801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,128,0,1,float16,fp8,0,2.920570691426595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,128,0,1,float16,fp8,0,1.3172372976938884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,128,0,1,float16,float16,0,1.3201226393381755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,128,0,1,fp8,fp8,0,0.9794399738311768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,128,0,1,fp8,fp8,0,0.9410453637441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,128,0,1,float16,float16,0,1.2999893029530842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,128,0,1,float16,fp8,0,1.3686134020487468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,128,0,1,fp8,fp8,0,1.0133013725280762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,128,0,1,float16,float16,0,1.3028746445973713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,128,0,1,float16,fp8,0,1.3362719217936199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,128,0,1,float16,float16,0,1.4794400533040364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,128,0,1,fp8,fp8,0,0.9503200054168701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,128,0,1,fp8,fp8,0,0.553002675374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,128,0,1,float16,float16,0,0.7405333518981934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,128,0,1,float16,fp8,0,1.3434240023295085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,128,0,1,float16,fp8,0,0.7790239651997884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,128,0,1,float16,float16,0,0.7601813475290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,128,0,1,float16,fp8,0,0.7295467058817545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,128,0,1,fp8,fp8,0,0.620416005452474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,128,0,1,float16,float16,0,0.71396803855896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,128,0,1,fp8,fp8,0,0.4990080197652181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,128,0,1,float16,fp8,0,0.7226719856262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,128,0,1,float16,float16,0,0.7098560333251953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,128,0,1,fp8,fp8,0,0.5022720098495483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,128,0,1,float16,fp8,0,0.715504010518392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,128,0,1,fp8,fp8,0,0.5009760061899821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,128,0,1,float16,float16,0,0.8330826759338379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,128,0,1,float16,fp8,0,0.7167572975158691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,128,0,1,fp8,fp8,0,7.618330637613933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,128,0,1,fp8,fp8,0,7.249130884806315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,128,0,1,float16,float16,0,10.475530624389648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,128,0,1,float16,fp8,0,10.147232055664062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,128,0,1,float16,float16,0,10.253957112630209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,128,0,1,float16,fp8,0,10.181253433227539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,128,0,1,float16,float16,0,10.292426427205404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,128,0,1,fp8,fp8,0,3.9545653661092124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,128,0,1,fp8,fp8,0,7.2135359446207685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,128,0,1,float16,fp8,0,5.541872024536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,128,0,1,float16,float16,0,5.514090855916341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,128,0,1,fp8,fp8,0,7.581216176350911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,128,0,1,float16,fp8,0,10.29527473449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,128,0,1,float16,float16,0,10.449733098347982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,128,0,1,fp8,fp8,0,3.7474985122680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,128,0,1,float16,float16,0,5.415632247924805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,128,0,1,float16,fp8,0,5.229850769042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,128,0,1,float16,fp8,0,10.9072634379069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,128,0,1,float16,float16,0,5.132778803507487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,128,0,1,fp8,fp8,0,3.716949462890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,128,0,1,float16,fp8,0,5.360031763712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,128,0,1,float16,float16,0,5.150634765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,128,0,1,fp8,fp8,0,3.6167093912760415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,128,0,1,float16,float16,0,2.832677205403646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,128,0,1,fp8,fp8,0,3.79036808013916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,128,0,1,float16,fp8,0,5.5117543538411455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,128,0,1,float16,float16,0,5.142021179199219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,128,0,1,fp8,fp8,0,1.9397493998209636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,128,0,1,float16,fp8,0,2.7670294443766275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,128,0,1,float16,fp8,0,5.581183751424153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,128,0,1,float16,float16,0,2.561253388722738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,128,0,1,fp8,fp8,0,1.734613259633382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,128,0,1,float16,fp8,0,2.618794600168864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,128,0,1,fp8,fp8,0,1.7309919993082683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,128,0,1,float16,float16,0,2.5381333033243814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,128,0,1,float16,fp8,0,2.5109333992004395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,128,0,1,float16,float16,0,2.6639199256896973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,128,0,1,fp8,fp8,0,1.9110666910807292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,128,0,1,float16,fp8,0,2.6930452982584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,128,0,1,float16,float16,0,1.433626651763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,128,0,1,float16,float16,0,2.709813435872396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,128,0,1,fp8,fp8,0,1.8103040059407551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,128,0,1,float16,fp8,0,1.3608320554097493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,128,0,1,fp8,fp8,0,1.0493706862131755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,128,0,1,float16,fp8,0,2.757370630900065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,128,0,1,float16,float16,0,1.3108426729838054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,128,0,1,float16,fp8,0,1.3442293802897136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,128,0,1,fp8,fp8,0,0.9491519927978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,128,0,1,fp8,fp8,0,0.9074772993723551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,128,0,1,float16,float16,0,1.3216160138448079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,128,0,1,float16,fp8,0,1.2666186491648357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,128,0,1,fp8,fp8,0,0.9566559791564941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,128,0,1,float16,float16,0,1.218618631362915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,128,0,1,float16,fp8,0,1.363653341929118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,128,0,1,float16,float16,0,1.2999253273010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,128,0,1,float16,fp8,0,1.306442658106486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,128,0,1,fp8,fp8,0,0.5352586507797241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,128,0,1,float16,float16,0,0.7452373504638672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,128,0,1,float16,float16,0,0.645962675412496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,128,0,1,fp8,fp8,0,0.9471306800842285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,128,0,1,float16,fp8,0,0.7242293357849121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,128,0,1,fp8,fp8,0,0.5239786704381307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,128,0,1,float16,fp8,0,0.7431253592173258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,128,0,1,float16,float16,0,0.6769386927286783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,128,0,1,fp8,fp8,0,0.5235199928283691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,128,0,1,float16,fp8,0,0.6557866732279459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,128,0,1,float16,float16,0,0.6652053197224935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,128,0,1,float16,fp8,0,0.6595893303553263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,128,0,1,fp8,fp8,0,0.5502133369445801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,128,0,1,float16,float16,0,0.4037066698074341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,128,0,1,float16,fp8,0,0.7253973484039307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,128,0,1,float16,float16,0,0.6692799727121989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,128,0,1,fp8,fp8,0,0.5221279859542847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,128,0,1,float16,fp8,0,0.41730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,128,0,1,float16,fp8,0,0.38626666863759357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,128,0,1,fp8,fp8,0,0.2967680096626282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,128,0,1,fp8,fp8,0,0.27321600914001465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,128,0,1,float16,float16,0,0.3767999807993571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,128,0,1,fp8,fp8,0,0.275818665822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,128,0,1,float16,float16,0,0.3964906533559163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,128,0,1,float16,float16,0,0.38228801886240643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,128,0,1,float16,fp8,0,0.3758560021718343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,128,0,1,float16,fp8,0,0.38389333089192706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,128,0,1,float16,float16,0,0.397216002146403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,128,0,1,float16,fp8,0,0.371509313583374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,128,0,1,fp8,fp8,0,0.2741066614786784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,128,0,1,fp8,fp8,0,0.2774133284886678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,128,0,1,fp8,fp8,0,4.647653261820476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,128,0,1,fp8,fp8,0,4.5474294026692705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,128,0,1,float16,float16,0,6.0879465738932295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,128,0,1,float16,fp8,0,6.278261184692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,128,0,1,float16,float16,0,6.381717046101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,128,0,1,float16,fp8,0,6.360501607259114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,128,0,1,float16,float16,0,6.373301188151042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,128,0,1,float16,fp8,0,6.319082895914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,128,0,1,fp8,fp8,0,4.628042538960774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,128,0,1,fp8,fp8,0,2.446901321411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,128,0,1,float16,float16,0,3.2422240575154624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,128,0,1,float16,fp8,0,3.2606614430745444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,128,0,1,fp8,fp8,0,4.6343199412028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,128,0,1,float16,float16,0,6.467109044392903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,128,0,1,float16,fp8,0,6.335669199625651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,128,0,1,fp8,fp8,0,2.229861259460449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,128,0,1,float16,float16,0,3.0918025970458984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,128,0,1,float16,fp8,0,3.1804428100585938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,128,0,1,float16,float16,0,3.01961612701416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,128,0,1,fp8,fp8,0,2.1792853673299155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,128,0,1,float16,fp8,0,3.0526933670043945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,128,0,1,fp8,fp8,0,2.246287981669108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,128,0,1,float16,float16,0,3.0642452239990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,128,0,1,float16,fp8,0,3.01145076751709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,128,0,1,float16,float16,0,3.007919947306315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,128,0,1,float16,float16,0,1.6227253278096516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,128,0,1,fp8,fp8,0,2.317258675893148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,128,0,1,float16,fp8,0,3.127178510030111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,128,0,1,float16,fp8,0,1.6832319895426433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,128,0,1,fp8,fp8,0,1.2397119998931885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,128,0,1,float16,float16,0,1.450943946838379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,128,0,1,fp8,fp8,0,1.1202720006306965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,128,0,1,float16,fp8,0,1.4478027025858562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,128,0,1,fp8,fp8,0,1.098410685857137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,128,0,1,float16,float16,0,1.4530773162841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,128,0,1,float16,fp8,0,1.4450133641560872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,128,0,1,float16,fp8,0,1.4585439364115398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,128,0,1,fp8,fp8,0,1.1383999983469646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,128,0,1,float16,float16,0,1.4833280245463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,128,0,1,float16,float16,0,1.504085381825765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,128,0,1,float16,float16,0,0.8461439609527588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,128,0,1,fp8,fp8,0,1.1203786532084148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,128,0,1,float16,fp8,0,1.5320320129394531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,128,0,1,float16,fp8,0,0.831770658493042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,128,0,1,fp8,fp8,0,0.6438186566034952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,128,0,1,float16,float16,0,0.7598079840342203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,128,0,1,fp8,fp8,0,0.6084373394648234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,128,0,1,float16,fp8,0,0.7660640080769857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,128,0,1,float16,float16,0,0.7630080382029215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,128,0,1,fp8,fp8,0,0.605135997136434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,128,0,1,float16,fp8,0,0.7708426316579183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,128,0,1,float16,float16,0,0.7567786375681559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,128,0,1,fp8,fp8,0,0.598037322362264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,128,0,1,float16,fp8,0,0.7576800187428793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,128,0,1,float16,float16,0,0.7780426343282064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,128,0,1,float16,float16,0,0.4447466532389323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,128,0,1,float16,fp8,0,0.7843786875406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,128,0,1,fp8,fp8,0,0.6018826564153036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,128,0,1,float16,fp8,0,0.4482400019963582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,128,0,1,fp8,fp8,0,0.35339732964833576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,128,0,1,float16,float16,0,0.4136586586634318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,128,0,1,float16,fp8,0,0.40701866149902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,128,0,1,fp8,fp8,0,0.3166559934616089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,128,0,1,float16,float16,0,0.4158613284428914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,128,0,1,float16,fp8,0,0.42444801330566406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,128,0,1,fp8,fp8,0,0.3120479981104533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,128,0,1,float16,float16,0,0.41858665148417157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,128,0,1,float16,fp8,0,0.4223039944966634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,128,0,1,fp8,fp8,0,0.31828800837198895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,128,0,1,fp8,fp8,0,0.31563733021418255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,128,0,1,float16,float16,0,0.4126720031102498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,128,0,1,float16,fp8,0,0.4230453173319499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,128,0,1,float16,float16,0,0.23647467295328775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,128,0,1,float16,fp8,0,0.23720532655715942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,128,0,1,fp8,fp8,0,0.198469340801239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,128,0,1,float16,float16,0,0.22184000412623087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,128,0,1,float16,fp8,0,0.2221333384513855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,128,0,1,fp8,fp8,0,0.18158932526906332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,128,0,1,float16,float16,0,0.21649066607157388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,128,0,1,float16,fp8,0,0.22325332959493002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,128,0,1,fp8,fp8,0,0.18201067050298056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,128,0,1,float16,float16,0,0.2215786576271057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,128,0,1,float16,fp8,0,0.22339733441670737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,128,0,1,fp8,fp8,0,0.18460800250371298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,128,0,1,float16,float16,0,0.22311999400456747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,128,0,1,float16,fp8,0,0.22543466091156006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,128,0,1,fp8,fp8,0,0.18017599980036417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,128,0,1,fp8,fp8,0,4.695733388264974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,128,0,1,fp8,fp8,0,4.873903910319011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,128,0,1,float16,float16,0,6.327023824055989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,128,0,1,float16,fp8,0,6.2608693440755205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,128,0,1,float16,fp8,0,6.293429056803386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,128,0,1,float16,float16,0,6.428784052530925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,128,0,1,float16,float16,0,6.303487777709961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,128,0,1,float16,fp8,0,6.432751973470052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,128,0,1,fp8,fp8,0,2.6921119689941406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,128,0,1,fp8,fp8,0,4.825573285420735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,128,0,1,float16,float16,0,3.0830345153808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,128,0,1,float16,float16,0,3.51908810933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,128,0,1,float16,fp8,0,3.4335734049479165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,128,0,1,fp8,fp8,0,4.938762664794922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,128,0,1,float16,float16,0,6.545797348022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,128,0,1,fp8,fp8,0,2.326752026875814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,128,0,1,float16,fp8,0,6.4715836842854815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,128,0,1,float16,fp8,0,3.140714645385742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,128,0,1,fp8,fp8,0,2.419520060221354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,128,0,1,float16,float16,0,3.1691627502441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,128,0,1,float16,fp8,0,3.131829261779785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,128,0,1,float16,float16,0,3.0934292475382485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,128,0,1,float16,fp8,0,3.210277239481608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,128,0,1,fp8,fp8,0,2.4099574089050293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,128,0,1,float16,float16,0,1.7189812660217285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,128,0,1,fp8,fp8,0,2.5066986083984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,128,0,1,float16,float16,0,3.1370827356974282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,128,0,1,float16,fp8,0,3.1548585891723633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,128,0,1,fp8,fp8,0,1.4035040537516277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,128,0,1,float16,fp8,0,1.7171999613444011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,128,0,1,float16,float16,0,1.4661706288655598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,128,0,1,float16,fp8,0,1.4762986501057942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,128,0,1,fp8,fp8,0,1.1867466767628987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,128,0,1,fp8,fp8,0,1.1857333183288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,128,0,1,float16,float16,0,1.4757173856099446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,128,0,1,float16,fp8,0,1.4869599342346191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,128,0,1,float16,float16,0,1.4658506711324055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,128,0,1,fp8,fp8,0,1.2039093176523845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,128,0,1,float16,fp8,0,1.4827893575032551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,128,0,1,float16,float16,0,1.549770673116048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,128,0,1,float16,float16,0,0.8711733023325602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,128,0,1,fp8,fp8,0,1.2356586456298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,128,0,1,fp8,fp8,0,0.7093013127644857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,128,0,1,float16,fp8,0,0.8635413646697998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,128,0,1,float16,fp8,0,1.536629358927409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,128,0,1,float16,float16,0,0.772874673207601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,128,0,1,float16,fp8,0,0.7635306517283121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,128,0,1,fp8,fp8,0,0.614677349726359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,128,0,1,float16,float16,0,0.7539520263671875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,128,0,1,fp8,fp8,0,0.6086879968643188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,128,0,1,float16,fp8,0,0.7595840295155843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,128,0,1,float16,float16,0,0.7701706886291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,128,0,1,float16,fp8,0,0.7688159942626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,128,0,1,fp8,fp8,0,0.6365866661071777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,128,0,1,float16,float16,0,0.7714933554331461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,128,0,1,float16,float16,0,0.4551946719487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,128,0,1,fp8,fp8,0,0.6246666510899862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,128,0,1,float16,fp8,0,0.7760586738586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,128,0,1,float16,fp8,0,0.45741331577301025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,128,0,1,float16,float16,0,0.40083734194437665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,128,0,1,fp8,fp8,0,0.33295466502507526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,128,0,1,fp8,fp8,0,0.3742133378982544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,128,0,1,float16,fp8,0,0.4111786683400472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,128,0,1,fp8,fp8,0,0.3359520037968953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,128,0,1,float16,float16,0,0.4056853453318278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,128,0,1,float16,fp8,0,0.398965318997701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,128,0,1,float16,float16,0,0.40852801005045575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,128,0,1,fp8,fp8,0,0.33765331904093426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,128,0,1,float16,fp8,0,0.4113333225250244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,128,0,1,float16,float16,0,0.40904533863067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,128,0,1,fp8,fp8,0,0.20454933245976767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,128,0,1,fp8,fp8,0,0.3434240023295085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,128,0,1,float16,float16,0,0.25059733788172406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,128,0,1,float16,fp8,0,0.23089067141215006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,128,0,1,float16,fp8,0,0.41544000307718915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,128,0,1,float16,fp8,0,0.2509760061899821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,128,0,1,float16,float16,0,0.22913599014282227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,128,0,1,fp8,fp8,0,0.17541333039601645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,128,0,1,float16,float16,0,0.23180800676345825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,128,0,1,float16,fp8,0,0.232095996538798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,128,0,1,fp8,fp8,0,0.17940799395243326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,128,0,1,float16,float16,0,0.23240000009536743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,128,0,1,float16,fp8,0,0.23470399777094522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,128,0,1,fp8,fp8,0,0.180074671904246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,128,0,1,float16,float16,0,0.23270400365193686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,128,0,1,float16,fp8,0,0.2333013415336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,128,0,1,float16,fp8,0,0.1269439955552419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,128,0,1,fp8,fp8,0,0.18338133891423544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,128,0,1,float16,float16,0,0.13426666458447775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,128,0,1,float16,fp8,0,0.136245330174764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,128,0,1,float16,fp8,0,0.12713066736857095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,128,0,1,fp8,fp8,0,0.10779733459154765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,128,0,1,fp8,fp8,0,0.11948800086975098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,128,0,1,float16,float16,0,0.12587199608484903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,128,0,1,fp8,fp8,0,0.10745066404342651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,128,0,1,float16,float16,0,0.12363200386365254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,128,0,1,float16,float16,0,0.12729600071907043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,128,0,1,float16,fp8,0,0.1272479991118113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,128,0,1,fp8,fp8,0,0.1099626620610555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,128,0,1,float16,float16,0,0.12839999794960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,128,0,1,float16,fp8,0,0.12852799892425537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,128,0,1,fp8,fp8,0,0.11147733529408772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,128,0,1,fp8,fp8,0,3.1875521341959634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,128,0,1,fp8,fp8,0,3.1892852783203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,128,0,1,float16,float16,0,3.9526294072469077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,128,0,1,float16,fp8,0,3.9415146509806314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,128,0,1,float16,float16,0,3.834303855895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,128,0,1,float16,fp8,0,3.968517303466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,128,0,1,float16,float16,0,4.002010663350423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,128,0,1,float16,fp8,0,4.042101224263509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,128,0,1,fp8,fp8,0,1.867077350616455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,128,0,1,float16,float16,0,2.2475199699401855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,128,0,1,float16,fp8,0,2.2191573778788247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,128,0,1,fp8,fp8,0,3.181269327799479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,128,0,1,float16,float16,0,1.9020105997721355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,128,0,1,fp8,fp8,0,3.22327454884847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,128,0,1,float16,float16,0,4.150501251220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,128,0,1,float16,fp8,0,4.095280011494954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,128,0,1,fp8,fp8,0,1.5918080012003581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,128,0,1,float16,fp8,0,1.9032212893168132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,128,0,1,float16,float16,0,1.9226400057474773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,128,0,1,fp8,fp8,0,1.5882293383280437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,128,0,1,float16,fp8,0,1.9067306518554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,128,0,1,float16,float16,0,1.975013256072998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,128,0,1,float16,fp8,0,1.9412906964619954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,128,0,1,fp8,fp8,0,1.6088053385416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,128,0,1,float16,float16,0,1.121498664220174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,128,0,1,float16,float16,0,1.9779787063598633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,128,0,1,fp8,fp8,0,1.6113386154174805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,128,0,1,float16,fp8,0,1.9901973406473796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,128,0,1,float16,float16,0,0.9207839965820312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,128,0,1,float16,fp8,0,1.1314559777577717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,128,0,1,fp8,fp8,0,0.9659679730733236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,128,0,1,float16,fp8,0,0.9391199747721354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,128,0,1,fp8,fp8,0,0.8203946749369303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,128,0,1,float16,float16,0,0.9289600054423014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,128,0,1,float16,fp8,0,0.9647520383199056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,128,0,1,fp8,fp8,0,0.7962453365325928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,128,0,1,float16,float16,0,0.9527786572774252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,128,0,1,fp8,fp8,0,0.8147679964701334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,128,0,1,float16,fp8,0,0.9486292997996012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,128,0,1,float16,float16,0,0.9594133694966634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,128,0,1,fp8,fp8,0,0.8343893686930338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,128,0,1,float16,fp8,0,0.96124800046285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,128,0,1,float16,float16,0,0.571066657702128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,128,0,1,float16,fp8,0,0.5781173308690389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,128,0,1,fp8,fp8,0,0.4904213349024455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,128,0,1,float16,float16,0,0.48662400245666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,128,0,1,float16,fp8,0,0.4951253334681193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,128,0,1,fp8,fp8,0,0.4254453182220459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,128,0,1,float16,fp8,0,0.49556267261505127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,128,0,1,float16,float16,0,0.4958560069402059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,128,0,1,fp8,fp8,0,0.4239413340886434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,128,0,1,float16,float16,0,0.48892800013224286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,128,0,1,float16,fp8,0,0.49916799863179523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,128,0,1,fp8,fp8,0,0.4272693395614624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,128,0,1,float16,float16,0,0.5013173421223959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,128,0,1,float16,fp8,0,0.5076533158620199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,128,0,1,fp8,fp8,0,0.4359360138575236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,128,0,1,float16,float16,0,0.3068319956461589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,128,0,1,float16,fp8,0,0.3047413428624471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,128,0,1,fp8,fp8,0,0.2691466609636943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,128,0,1,float16,float16,0,0.26766933997472125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,128,0,1,float16,fp8,0,0.2679520050684611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,128,0,1,fp8,fp8,0,0.21761600176493326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,128,0,1,float16,float16,0,0.2696160078048706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,128,0,1,fp8,fp8,0,0.21765865882237753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,128,0,1,float16,fp8,0,0.2714560031890869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,128,0,1,float16,float16,0,0.27083200216293335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,128,0,1,float16,fp8,0,0.2708853284517924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,128,0,1,float16,float16,0,0.1687893271446228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,128,0,1,fp8,fp8,0,0.22062933444976807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,128,0,1,float16,float16,0,0.2730240027109782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,128,0,1,float16,fp8,0,0.2773173252741496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,128,0,1,fp8,fp8,0,0.22595733404159546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,128,0,1,float16,fp8,0,0.16908266146977743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,128,0,1,fp8,fp8,0,0.14284267028172812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,128,0,1,fp8,fp8,0,0.12582932909329733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,128,0,1,float16,float16,0,0.14292266964912415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,128,0,1,float16,fp8,0,0.14638933539390564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,128,0,1,fp8,fp8,0,0.12468266487121582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,128,0,1,float16,float16,0,0.14406933387120566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,128,0,1,float16,fp8,0,0.14731199542681375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,128,0,1,float16,float16,0,0.14543466766675314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,128,0,1,float16,float16,0,0.0936906635761261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,128,0,1,float16,fp8,0,0.14688533544540405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,128,0,1,fp8,fp8,0,0.12733866771062216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,128,0,1,float16,float16,0,0.1454080045223236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,128,0,1,float16,fp8,0,0.15015467007954916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,128,0,1,fp8,fp8,0,0.12826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,128,0,1,float16,fp8,0,0.09570133686065674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,128,0,1,fp8,fp8,0,0.08569600184758504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,128,0,1,float16,float16,0,0.08832533160845439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,128,0,1,float16,fp8,0,0.08868267138799031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,128,0,1,fp8,fp8,0,0.07684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,128,0,1,fp8,fp8,0,0.07840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,128,0,1,float16,float16,0,0.08725866675376892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,128,0,1,float16,fp8,0,0.08954133590062459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,128,0,1,fp8,fp8,0,0.07819733520348866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,128,0,1,float16,float16,0,0.08923199772834778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,128,0,1,float16,fp8,0,0.08834667007128398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,128,0,1,float16,float16,0,0.08889066179593404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,128,0,1,float16,fp8,0,0.09057600299517314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,128,0,1,fp8,fp8,0,0.08131200075149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,128,0,1,fp8,fp8,0,3.3832321166992188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,128,0,1,float16,float16,0,3.9157654444376626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,128,0,1,float16,fp8,0,4.030192057291667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,128,0,1,float16,float16,0,3.9542665481567383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,128,0,1,fp8,fp8,0,3.539306640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,128,0,1,float16,fp8,0,4.045066515604655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,128,0,1,float16,float16,0,4.024613380432129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,128,0,1,float16,fp8,0,4.035285313924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,128,0,1,fp8,fp8,0,3.531855901082357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,128,0,1,float16,float16,0,2.5425546964009604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,128,0,1,float16,float16,0,4.133754730224609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,128,0,1,float16,float16,0,1.9278133710225422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,128,0,1,float16,fp8,0,2.504858652750651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,128,0,1,fp8,fp8,0,2.1507040659586587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,128,0,1,float16,fp8,0,4.268453280131022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,128,0,1,fp8,fp8,0,3.7725280125935874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,128,0,1,float16,fp8,0,1.9366559982299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,128,0,1,fp8,fp8,0,1.7340480486551921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,128,0,1,float16,float16,0,1.9524854024251301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,128,0,1,float16,fp8,0,1.942021369934082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,128,0,1,fp8,fp8,0,1.780783971150716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,128,0,1,float16,float16,0,2.02677329381307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,128,0,1,float16,fp8,0,1.961125373840332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,128,0,1,fp8,fp8,0,1.8106187184651692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,128,0,1,float16,float16,0,1.2815093199412029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,128,0,1,float16,float16,0,2.026085376739502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,128,0,1,fp8,fp8,0,1.0805813471476238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,128,0,1,float16,fp8,0,1.254266659418742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,128,0,1,fp8,fp8,0,1.8809119860331218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,128,0,1,float16,fp8,0,2.045157273610433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,128,0,1,float16,float16,0,0.9823893706003824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,128,0,1,float16,fp8,0,0.9833386739095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,128,0,1,fp8,fp8,0,0.86408003171285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,128,0,1,float16,float16,0,0.9860693613688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,128,0,1,float16,fp8,0,0.9988640149434408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,128,0,1,fp8,fp8,0,0.8882559935251871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,128,0,1,float16,float16,0,1.004698673884074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,128,0,1,float16,fp8,0,0.9939466317494711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,128,0,1,fp8,fp8,0,0.9142293135325114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,128,0,1,float16,float16,0,1.0249439875284831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,128,0,1,float16,float16,0,0.6483039855957031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,128,0,1,float16,fp8,0,0.6276266574859619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,128,0,1,fp8,fp8,0,0.9511626561482748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,128,0,1,float16,fp8,0,1.0188746452331543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,128,0,1,float16,float16,0,0.506330649058024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,128,0,1,fp8,fp8,0,0.5511360168457031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,128,0,1,fp8,fp8,0,0.43586134910583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,128,0,1,float16,fp8,0,0.5068586667378744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,128,0,1,float16,float16,0,0.5089493195215861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,128,0,1,float16,float16,0,0.5140266815821329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,128,0,1,float16,fp8,0,0.5093173185984293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,128,0,1,fp8,fp8,0,0.4503413438796997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,128,0,1,float16,fp8,0,0.5115893284479777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,128,0,1,fp8,fp8,0,0.45691200097401935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,128,0,1,float16,float16,0,0.5276000102361044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,128,0,1,float16,fp8,0,0.5233866771062216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,128,0,1,fp8,fp8,0,0.4832640091578166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,128,0,1,float16,float16,0,0.33885331948598224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,128,0,1,float16,fp8,0,0.32839999596277875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,128,0,1,fp8,fp8,0,0.28235199054082233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,128,0,1,float16,float16,0,0.265669325987498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,128,0,1,float16,float16,0,0.26546667019526166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,128,0,1,float16,fp8,0,0.26497600475947064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,128,0,1,fp8,fp8,0,0.23265600204467773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,128,0,1,float16,fp8,0,0.2683626612027486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,128,0,1,float16,fp8,0,0.2685546676317851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,128,0,1,fp8,fp8,0,0.23712533712387085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,128,0,1,float16,float16,0,0.268394668896993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,128,0,1,float16,float16,0,0.27633599440256756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,128,0,1,fp8,fp8,0,0.23688000440597534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,128,0,1,fp8,fp8,0,0.24893865982691446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,128,0,1,float16,float16,0,0.18222399552663168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,128,0,1,float16,fp8,0,0.1425226628780365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,128,0,1,float16,fp8,0,0.2743893265724182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,128,0,1,float16,fp8,0,0.17565866311391196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,128,0,1,fp8,fp8,0,0.145306666692098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,128,0,1,float16,float16,0,0.14266666769981384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,128,0,1,fp8,fp8,0,0.11294933160146077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,128,0,1,float16,float16,0,0.14603199561436972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,128,0,1,float16,fp8,0,0.14698666334152222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,128,0,1,fp8,fp8,0,0.11507200201352437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,128,0,1,float16,float16,0,0.14724799990653992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,128,0,1,float16,fp8,0,0.14763200283050537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,128,0,1,fp8,fp8,0,0.11699733138084412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,128,0,1,float16,float16,0,0.15056533614794412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,128,0,1,float16,fp8,0,0.15026133259137472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,128,0,1,fp8,fp8,0,0.12040000160535176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,128,0,1,float16,float16,0,0.09713066617647807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,128,0,1,float16,fp8,0,0.09422933061917622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,128,0,1,fp8,fp8,0,0.08021866778532664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,128,0,1,float16,float16,0,0.07554133236408234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,128,0,1,float16,float16,0,0.07735999921957652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,128,0,1,float16,fp8,0,0.07549333572387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,128,0,1,fp8,fp8,0,0.06292800108591716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,128,0,1,float16,float16,0,0.07562666634718578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,128,0,1,float16,fp8,0,0.07548800110816956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,128,0,1,fp8,fp8,0,0.06482133269309998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,128,0,1,float16,fp8,0,0.07613333563009898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,128,0,1,float16,fp8,0,0.0498933345079422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,128,0,1,fp8,fp8,0,0.0652159998814265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,128,0,1,float16,float16,0,0.07921066880226135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,128,0,1,float16,fp8,0,0.07934399942557017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,128,0,1,fp8,fp8,0,0.06795733173688252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,128,0,1,float16,float16,0,0.05072000126043955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,128,0,1,fp8,fp8,0,0.045365333557128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,128,0,1,float16,float16,0,0.04299733539422353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,128,0,1,float16,fp8,0,0.043103997906049095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,128,0,1,float16,fp8,0,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,128,0,1,fp8,fp8,0,0.03679466744263967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,128,0,1,float16,float16,0,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,128,0,1,float16,fp8,0,0.043578664461771645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,128,0,1,fp8,fp8,0,0.037647999823093414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,128,0,1,float16,float16,0,0.04355733096599579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,128,0,1,fp8,fp8,0,0.03629866739114126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,128,0,1,float16,float16,0,0.044400001565615334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,128,0,1,float16,fp8,0,0.04509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,128,0,1,fp8,fp8,0,0.039274667700131737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,128,0,1,float16,float16,0,2.9934666951497397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,128,0,1,fp8,fp8,0,2.7902825673421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,128,0,1,float16,fp8,0,3.0083465576171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,128,0,1,float16,float16,0,3.028239885965983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,128,0,1,float16,fp8,0,3.0235093434651694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,128,0,1,fp8,fp8,0,2.860330581665039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,128,0,1,float16,float16,0,3.051919937133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,128,0,1,float16,float16,0,2.136650721232096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,128,0,1,float16,fp8,0,2.0557653109232583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,128,0,1,float16,fp8,0,3.0132481257120767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,128,0,1,fp8,fp8,0,2.91046937306722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,128,0,1,float16,float16,0,3.179775873819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,128,0,1,float16,fp8,0,3.182688077290853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,128,0,1,fp8,fp8,0,3.055882771809896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,128,0,1,float16,float16,0,1.5120800336201985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,128,0,1,float16,fp8,0,1.5020853678385417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,128,0,1,fp8,fp8,0,1.3916319211324055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,128,0,1,fp8,fp8,0,1.8315040270487468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,128,0,1,float16,float16,0,1.4959039688110352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,128,0,1,float16,fp8,0,1.5254507064819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,128,0,1,fp8,fp8,0,1.414240042368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,128,0,1,float16,float16,0,1.5457065900166829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,128,0,1,float16,fp8,0,1.531434694925944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,128,0,1,fp8,fp8,0,1.463098684946696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,128,0,1,float16,float16,0,1.6004427274068196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,128,0,1,float16,float16,0,1.0727466742197673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,128,0,1,float16,fp8,0,1.5906079610188801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,128,0,1,fp8,fp8,0,1.556821346282959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,128,0,1,fp8,fp8,0,0.9234879811604818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,128,0,1,float16,float16,0,0.7630666891733805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,128,0,1,float16,fp8,0,1.0332053502400715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,128,0,1,float16,fp8,0,0.7714506785074869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,128,0,1,float16,fp8,0,0.7632160186767578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,128,0,1,fp8,fp8,0,0.6941226323445638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,128,0,1,fp8,fp8,0,0.721946636835734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,128,0,1,float16,float16,0,0.7680959701538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,128,0,1,float16,float16,0,0.7800213495890299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,128,0,1,float16,fp8,0,0.7750666936238607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,128,0,1,fp8,fp8,0,0.7482933203379313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,128,0,1,float16,float16,0,0.8148319721221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,128,0,1,float16,fp8,0,0.8037066459655762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,128,0,1,fp8,fp8,0,0.7839413483937582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,128,0,1,float16,float16,0,0.5434666474660238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,128,0,1,float16,fp8,0,0.5171093146006266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,128,0,1,fp8,fp8,0,0.46562667687733966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,128,0,1,float16,float16,0,0.3926773468653361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,128,0,1,float16,float16,0,0.3953760067621867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,128,0,1,float16,fp8,0,0.3939093351364136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,128,0,1,fp8,fp8,0,0.35672001043955487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,128,0,1,float16,fp8,0,0.3948959906895955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,128,0,1,fp8,fp8,0,0.36432000001271564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,128,0,1,float16,float16,0,0.39924800395965576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,128,0,1,float16,fp8,0,0.39979199568430585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,128,0,1,fp8,fp8,0,0.36724801858266193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,128,0,1,float16,float16,0,0.41831998030344647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,128,0,1,float16,float16,0,0.28168533245722455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,128,0,1,float16,fp8,0,0.4115999937057495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,128,0,1,fp8,fp8,0,0.40134934584299725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,128,0,1,float16,fp8,0,0.27242666482925415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,128,0,1,fp8,fp8,0,0.2378879984219869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,128,0,1,float16,float16,0,0.2070186734199524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,128,0,1,float16,fp8,0,0.20670400063196817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,128,0,1,float16,float16,0,0.21379733085632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,128,0,1,fp8,fp8,0,0.18823999166488647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,128,0,1,float16,fp8,0,0.20813333988189697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,128,0,1,float16,float16,0,0.20722667376200357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,128,0,1,fp8,fp8,0,0.19255467255910239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,128,0,1,float16,fp8,0,0.20895999670028687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,128,0,1,fp8,fp8,0,0.19305066267649332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,128,0,1,float16,float16,0,0.21949867407480875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,128,0,1,float16,fp8,0,0.2173759937286377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,128,0,1,fp8,fp8,0,0.20338666439056396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,128,0,1,fp8,fp8,0,0.095360000928243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,128,0,1,float16,float16,0,0.1509173313776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,128,0,1,float16,fp8,0,0.14732799927393594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,128,0,1,fp8,fp8,0,0.12854400277137756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,128,0,1,float16,float16,0,0.1169706682364146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,128,0,1,float16,float16,0,0.10940800110499065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,128,0,1,float16,fp8,0,0.1122773289680481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,128,0,1,float16,float16,0,0.11400000254313152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,128,0,1,float16,fp8,0,0.11343999703725179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,128,0,1,fp8,fp8,0,0.09724799791971843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,128,0,1,float16,fp8,0,0.11575999855995178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,128,0,1,float16,fp8,0,0.08403199911117554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,128,0,1,fp8,fp8,0,0.09935999910036723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,128,0,1,float16,float16,0,0.11875733733177185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,128,0,1,float16,fp8,0,0.11745599905649821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,128,0,1,fp8,fp8,0,0.10130133231480916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,128,0,1,float16,float16,0,0.08737066388130188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,128,0,1,float16,fp8,0,0.061610668897628784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,128,0,1,fp8,fp8,0,0.07178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,128,0,1,float16,float16,0,0.06131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,128,0,1,float16,fp8,0,0.06168533364931742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,128,0,1,fp8,fp8,0,0.05298133194446564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,128,0,1,float16,float16,0,0.06259199976921082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,128,0,1,fp8,fp8,0,0.0543039987484614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,128,0,1,float16,float16,0,0.06341866652170818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,128,0,1,float16,fp8,0,0.0626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,128,0,1,fp8,fp8,0,0.055914665261904396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,128,0,1,float16,float16,0,0.06547733147939046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,128,0,1,float16,fp8,0,0.06531199812889099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,128,0,1,fp8,fp8,0,0.05821333328882853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,128,0,1,float16,float16,0,0.044064000248909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,128,0,1,float16,fp8,0,0.043285335103670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,128,0,1,fp8,fp8,0,0.04211199780305227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,128,0,1,float16,float16,0,0.0372533326347669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,128,0,1,float16,fp8,0,0.03714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,128,0,1,fp8,fp8,0,0.031983998914559685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,128,0,1,float16,float16,0,0.03702933341264725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,128,0,1,float16,fp8,0,0.03687999894221624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,128,0,1,fp8,fp8,0,0.03151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,128,0,1,float16,float16,0,0.0371573343873024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,128,0,1,float16,fp8,0,0.037765334049860634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,128,0,1,fp8,fp8,0,0.032229334115982056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,128,0,1,float16,float16,0,0.03804266701141993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,128,0,1,float16,fp8,0,0.03829866647720337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,128,0,1,fp8,fp8,0,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,128,0,1,float16,float16,0,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,128,0,1,float16,fp8,0,0.02703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,128,0,1,fp8,fp8,0,0.02462399999300639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,128,0,1,float16,float16,0,0.025946666797002155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,128,0,1,float16,fp8,0,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,128,0,1,fp8,fp8,0,0.023498666783173878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,128,0,1,float16,float16,0,0.02610666553179423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,128,0,1,float16,fp8,0,0.025888000925381977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,128,0,1,fp8,fp8,0,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,128,0,1,float16,float16,0,0.02640533447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,128,0,1,float16,fp8,0,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,128,0,1,fp8,fp8,0,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,128,0,1,float16,fp8,0,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,128,0,1,fp8,fp8,0,0.024080000817775726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,128,0,1,float16,float16,0,1.2886239687601726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,128,0,1,float16,fp8,0,1.2928746541341145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,128,0,1,fp8,fp8,0,1.1037387053171794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,128,0,1,float16,float16,0,1.303594668706258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,128,0,1,fp8,fp8,0,1.1443413098653157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,128,0,1,float16,fp8,0,1.3013973236083984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,128,0,1,float16,float16,0,1.328879992167155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,128,0,1,fp8,fp8,0,1.1857919692993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,128,0,1,float16,fp8,0,1.3162079652150471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,128,0,1,fp8,fp8,0,0.7910613218943278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,128,0,1,float16,float16,0,1.386954625447591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,128,0,1,float16,fp8,0,0.9376746813456217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,128,0,1,float16,float16,0,0.9856266975402832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,128,0,1,float16,fp8,0,1.3728426297505696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,128,0,1,fp8,fp8,0,1.2712639967600505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,128,0,1,float16,float16,0,0.65502401192983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,128,0,1,fp8,fp8,0,0.5487146774927775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,128,0,1,float16,fp8,0,0.6585226853688558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,128,0,1,float16,fp8,0,0.6540000041325887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,128,0,1,float16,float16,0,0.6590773264567057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,128,0,1,fp8,fp8,0,0.5715306599934896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,128,0,1,float16,float16,0,0.6758879820505778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,128,0,1,float16,fp8,0,0.6681973139444987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,128,0,1,fp8,fp8,0,0.5885013341903687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,128,0,1,float16,float16,0,0.7050560315450033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,128,0,1,float16,fp8,0,0.695898691813151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,128,0,1,float16,float16,0,0.4975359837214152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,128,0,1,float16,fp8,0,0.47820266087849933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,128,0,1,float16,fp8,0,0.34089601039886475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,128,0,1,fp8,fp8,0,0.6400533517201742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,128,0,1,fp8,fp8,0,0.40698667367299396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,128,0,1,fp8,fp8,0,0.284223993619283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,128,0,1,float16,float16,0,0.3378346761067708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,128,0,1,float16,float16,0,0.34080533186594647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,128,0,1,float16,fp8,0,0.3420799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,128,0,1,float16,float16,0,0.34724799791971844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,128,0,1,fp8,fp8,0,0.29254400730133057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,128,0,1,float16,float16,0,0.3654293219248454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,128,0,1,fp8,fp8,0,0.30409600337346393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,128,0,1,float16,fp8,0,0.3463040192921956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,128,0,1,float16,fp8,0,0.35734931627909344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,128,0,1,fp8,fp8,0,0.33125332991282147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,128,0,1,float16,float16,0,0.26019734144210815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,128,0,1,fp8,fp8,0,0.15129066507021585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,128,0,1,float16,fp8,0,0.2516533335049947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,128,0,1,fp8,fp8,0,0.20745599269866943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,128,0,1,float16,float16,0,0.17929067214330038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,128,0,1,float16,fp8,0,0.17999466260274252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,128,0,1,float16,float16,0,0.18009066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,128,0,1,float16,fp8,0,0.1812480092048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,128,0,1,fp8,fp8,0,0.15743999679883322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,128,0,1,float16,float16,0,0.18262400229771933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,128,0,1,float16,fp8,0,0.18528532981872559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,128,0,1,fp8,fp8,0,0.15834133823712668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,128,0,1,float16,float16,0,0.1920373241106669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,128,0,1,float16,fp8,0,0.19104532400767008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,128,0,1,fp8,fp8,0,0.1660160024960836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,128,0,1,float16,float16,0,0.14273599783579508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,128,0,1,float16,fp8,0,0.1369439959526062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,128,0,1,fp8,fp8,0,0.11873066425323486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,128,0,1,float16,float16,0,0.09675199786822002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,128,0,1,float16,fp8,0,0.09670399626096089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,128,0,1,fp8,fp8,0,0.0865119993686676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,128,0,1,float16,float16,0,0.099263995885849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,128,0,1,float16,fp8,0,0.09839999675750732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,128,0,1,fp8,fp8,0,0.08860799670219421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,128,0,1,float16,fp8,0,0.10548266768455505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,128,0,1,float16,float16,0,0.10191999872525533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,128,0,1,float16,fp8,0,0.10085333387056987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,128,0,1,float16,float16,0,0.08567466338475545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,128,0,1,fp8,fp8,0,0.09060266613960266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,128,0,1,float16,float16,0,0.055349335074424744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,128,0,1,float16,float16,0,0.10445333520571391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,128,0,1,fp8,fp8,0,0.0920853316783905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,128,0,1,float16,float16,0,0.05570666491985321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,128,0,1,float16,fp8,0,0.08238400022188823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,128,0,1,fp8,fp8,0,0.06657066444555919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,128,0,1,float16,fp8,0,0.05715733269850413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,128,0,1,fp8,fp8,0,0.0487360010544459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,128,0,1,float16,fp8,0,0.05635199944178263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,128,0,1,fp8,fp8,0,0.04897599915663401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,128,0,1,float16,float16,0,0.05719999969005585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,128,0,1,float16,fp8,0,0.05716800192991892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,128,0,1,fp8,fp8,0,0.05022400120894114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,128,0,1,float16,float16,0,0.059674665331840515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,128,0,1,float16,fp8,0,0.06028266747792562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,128,0,1,fp8,fp8,0,0.0529120018084844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,128,0,1,float16,float16,0,0.04068800061941147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,128,0,1,fp8,fp8,0,0.02942399928967158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,128,0,1,float16,fp8,0,0.039594667653242745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,128,0,1,fp8,fp8,0,0.0388373335202535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,128,0,1,fp8,fp8,0,0.029493334392706554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,128,0,1,float16,float16,0,0.03408000121514002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,128,0,1,float16,fp8,0,0.034346667428811394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,128,0,1,float16,float16,0,0.03401066611210505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,128,0,1,float16,float16,0,0.035071998834609985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,128,0,1,float16,fp8,0,0.03377600014209747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,128,0,1,fp8,fp8,0,0.03092266619205475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,128,0,1,float16,float16,0,0.034058667719364166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,128,0,1,float16,fp8,0,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,128,0,1,fp8,fp8,0,0.022783999641736347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,128,0,1,float16,fp8,0,0.0340639998515447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,128,0,1,fp8,fp8,0,0.02980799973011017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,128,0,1,float16,fp8,0,0.035360001027584076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,128,0,1,float16,float16,0,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,128,0,1,float16,float16,0,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,128,0,1,float16,float16,0,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,128,0,1,float16,fp8,0,0.02350933353106181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,128,0,1,fp8,fp8,0,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,128,0,1,float16,fp8,0,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,128,0,1,fp8,fp8,0,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,128,0,1,float16,float16,0,0.023733332753181458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,128,0,1,fp8,fp8,0,0.022111999491850536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,128,0,1,float16,fp8,0,0.024090667565663654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,128,0,1,fp8,fp8,0,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,128,0,1,float16,float16,0,0.02422933280467987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,128,0,1,float16,fp8,0,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,128,0,1,float16,float16,0,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,128,0,1,float16,fp8,0,0.02075733368595441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,128,0,1,fp8,fp8,0,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,128,0,1,float16,float16,0,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,128,0,1,float16,fp8,0,0.020117333779732387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,128,0,1,fp8,fp8,0,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,128,0,1,float16,fp8,0,0.02013333390156428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,128,0,1,float16,float16,0,0.01977066695690155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,128,0,1,float16,fp8,0,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,128,0,1,fp8,fp8,0,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,128,0,1,float16,float16,0,0.019738666713237762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,128,0,1,fp8,fp8,0,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,128,0,1,float16,float16,0,0.020074666788180668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,128,0,1,float16,fp8,0,0.02045866722861926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,128,0,1,fp8,fp8,0,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,128,0,1,float16,float16,0,0.6235946814219157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,128,0,1,float16,fp8,0,0.6233013470967611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,128,0,1,fp8,fp8,0,0.5482240120569865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,128,0,1,float16,float16,0,0.6267360051472982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,128,0,1,float16,fp8,0,0.6244906584421793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,128,0,1,fp8,fp8,0,0.5770826737085978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,128,0,1,float16,float16,0,0.6422293186187744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,128,0,1,float16,fp8,0,0.6402133305867513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,128,0,1,fp8,fp8,0,0.5997759898503622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,128,0,1,float16,float16,0,0.6751893361409506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,128,0,1,float16,fp8,0,0.6650613149007162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,128,0,1,float16,float16,0,0.5071093241373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,128,0,1,float16,fp8,0,0.3221919933954875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,128,0,1,fp8,fp8,0,0.3931359847386678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,128,0,1,fp8,fp8,0,0.645029346148173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,128,0,1,float16,fp8,0,0.4896320104598999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,128,0,1,float16,float16,0,0.3214720090230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,128,0,1,fp8,fp8,0,0.28064000606536865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,128,0,1,float16,float16,0,0.3251466751098633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,128,0,1,fp8,fp8,0,0.2909173369407654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,128,0,1,float16,fp8,0,0.3294186592102051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,128,0,1,float16,fp8,0,0.3247306744257609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,128,0,1,float16,float16,0,0.33179734150568646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,128,0,1,fp8,fp8,0,0.3041706681251526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,128,0,1,float16,float16,0,0.346837321917216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,128,0,1,float16,fp8,0,0.343941330909729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,128,0,1,float16,float16,0,0.2671893239021301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,128,0,1,float16,fp8,0,0.2579786578814189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,128,0,1,fp8,fp8,0,0.32713599999745685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,128,0,1,fp8,fp8,0,0.20535467068354288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,128,0,1,float16,float16,0,0.17013333241144815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,128,0,1,float16,fp8,0,0.17022399107615152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,128,0,1,fp8,fp8,0,0.1506186624368032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,128,0,1,float16,float16,0,0.17281067371368408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,128,0,1,float16,fp8,0,0.17360534270604452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,128,0,1,fp8,fp8,0,0.15345066785812378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,128,0,1,float16,float16,0,0.1768959959348043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,128,0,1,float16,fp8,0,0.17667200167973837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,128,0,1,fp8,fp8,0,0.15945067008336386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,128,0,1,float16,float16,0,0.18578133980433145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,128,0,1,float16,fp8,0,0.18626133600870767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,128,0,1,fp8,fp8,0,0.167087992032369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,128,0,1,float16,float16,0,0.1451573371887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,128,0,1,float16,fp8,0,0.1420746644337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,128,0,1,fp8,fp8,0,0.11405866344769795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,128,0,1,float16,float16,0,0.09243200222651164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,128,0,1,float16,fp8,0,0.09199999769528706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,128,0,1,fp8,fp8,0,0.08497066299120586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,128,0,1,float16,float16,0,0.09542933106422424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,128,0,1,float16,fp8,0,0.09481066465377808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,128,0,1,fp8,fp8,0,0.08669867118199666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,128,0,1,float16,float16,0,0.09815999865531921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,128,0,1,float16,fp8,0,0.0986400047938029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,128,0,1,fp8,fp8,0,0.08789867162704468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,128,0,1,float16,float16,0,0.10475732882817586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,128,0,1,float16,fp8,0,0.10148800412813823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,128,0,1,fp8,fp8,0,0.09132267038027446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,128,0,1,float16,float16,0,0.0822026679913203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,128,0,1,float16,fp8,0,0.07977066437403361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,128,0,1,float16,float16,0,0.05366399884223938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,128,0,1,fp8,fp8,0,0.0672106643517812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,128,0,1,float16,float16,0,0.0529120018084844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,128,0,1,float16,fp8,0,0.05458133419354757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,128,0,1,fp8,fp8,0,0.047050664822260536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,128,0,1,float16,fp8,0,0.054655998945236206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,128,0,1,fp8,fp8,0,0.04844800134499868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,128,0,1,float16,float16,0,0.05489600201447805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,128,0,1,fp8,fp8,0,0.05261866748332977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,128,0,1,float16,fp8,0,0.05517866710821787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,128,0,1,fp8,fp8,0,0.04996799925963084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,128,0,1,float16,float16,0,0.058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,128,0,1,float16,fp8,0,0.057434668143590294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,128,0,1,float16,float16,0,0.040720000863075256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,128,0,1,float16,fp8,0,0.03864533454179764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,128,0,1,fp8,fp8,0,0.03846933444341024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,128,0,1,float16,float16,0,0.033370666205883026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,128,0,1,float16,fp8,0,0.033471999069054924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,128,0,1,float16,float16,0,0.03372266640265783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,128,0,1,float16,fp8,0,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,128,0,1,fp8,fp8,0,0.029189333319664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,128,0,1,float16,float16,0,0.03323733309904734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,128,0,1,float16,fp8,0,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,128,0,1,fp8,fp8,0,0.02938666691382726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,128,0,1,fp8,fp8,0,0.029792000850041706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,128,0,1,float16,float16,0,0.034074666599432625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,128,0,1,float16,fp8,0,0.033904001116752625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,128,0,1,float16,float16,0,0.02260799954334895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,128,0,1,fp8,fp8,0,0.031066666046778362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,128,0,1,float16,float16,0,0.023711999257405598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,128,0,1,float16,fp8,0,0.024175999065240223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,128,0,1,fp8,fp8,0,0.02276266614596049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,128,0,1,float16,fp8,0,0.02223466585079829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,128,0,1,fp8,fp8,0,0.020432000358899433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,128,0,1,fp8,fp8,0,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,128,0,1,float16,float16,0,0.022245332598686218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,128,0,1,float16,fp8,0,0.022442666192849476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,128,0,1,fp8,fp8,0,0.0205226664741834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,128,0,1,float16,float16,0,0.022522665560245514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,128,0,1,float16,fp8,0,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,128,0,1,float16,float16,0,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,128,0,1,float16,fp8,0,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,128,0,1,fp8,fp8,0,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,128,0,1,float16,float16,0,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,128,0,1,float16,fp8,0,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,128,0,1,fp8,fp8,0,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,128,0,1,float16,float16,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,128,0,1,float16,fp8,0,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,128,0,1,fp8,fp8,0,0.01829333355029424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,128,0,1,float16,float16,0,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,128,0,1,float16,fp8,0,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,128,0,1,fp8,fp8,0,0.01817600056529045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,128,0,1,float16,float16,0,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,128,0,1,float16,fp8,0,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,128,0,1,fp8,fp8,0,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,128,0,1,float16,float16,0,0.01836266616980235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,128,0,1,float16,fp8,0,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,128,0,1,fp8,fp8,0,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,128,0,1,float16,float16,0,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,128,0,1,float16,fp8,0,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,128,0,1,fp8,fp8,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,128,0,1,float16,float16,0,0.016565332810084026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,128,0,1,float16,fp8,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,128,0,1,fp8,fp8,0,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,128,0,1,float16,float16,0,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,128,0,1,float16,fp8,0,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,128,0,1,fp8,fp8,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,128,0,1,float16,float16,0,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,128,0,1,float16,fp8,0,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,128,0,1,fp8,fp8,0,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,128,0,1,float16,float16,0,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,128,0,1,float16,fp8,0,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,128,0,1,fp8,fp8,0,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,128,0,1,float16,float16,0,0.3626399834950765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,128,0,1,float16,float16,0,0.36084266503651935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,128,0,1,float16,fp8,0,0.3613119920094808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,128,0,1,fp8,fp8,0,0.3905706803003947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,128,0,1,float16,fp8,0,0.36393598715464276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,128,0,1,fp8,fp8,0,0.4017173449198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,128,0,1,float16,float16,0,0.37438400586446124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,128,0,1,float16,fp8,0,0.36951998869578045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,128,0,1,fp8,fp8,0,0.4089333216349284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,128,0,1,float16,float16,0,0.38899731636047363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,128,0,1,float16,fp8,0,0.3834986686706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,128,0,1,float16,float16,0,0.26799466212590534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,128,0,1,float16,fp8,0,0.2600906689961751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,128,0,1,fp8,fp8,0,0.43516798814137775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,128,0,1,fp8,fp8,0,0.25723199049631756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,128,0,1,float16,fp8,0,0.18643200397491455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,128,0,1,float16,float16,0,0.18701332807540894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,128,0,1,fp8,fp8,0,0.20589866240819296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,128,0,1,float16,float16,0,0.18918933471043906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,128,0,1,float16,fp8,0,0.1902880072593689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,128,0,1,fp8,fp8,0,0.2107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,128,0,1,float16,float16,0,0.19339199860890707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,128,0,1,float16,fp8,0,0.19311465819676718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,128,0,1,fp8,fp8,0,0.21339199940363565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,128,0,1,float16,float16,0,0.2033066749572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,128,0,1,float16,fp8,0,0.13954133788744608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,128,0,1,fp8,fp8,0,0.13897066315015158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,128,0,1,float16,fp8,0,0.20148267348607382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,128,0,1,fp8,fp8,0,0.22056533892949423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,128,0,1,float16,float16,0,0.14299733440081278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,128,0,1,float16,float16,0,0.09868799646695454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,128,0,1,float16,fp8,0,0.09859200318654378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,128,0,1,fp8,fp8,0,0.11238933602968852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,128,0,1,float16,float16,0,0.10066133737564087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,128,0,1,float16,fp8,0,0.10391466816266377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,128,0,1,fp8,fp8,0,0.11573333541552226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,128,0,1,float16,fp8,0,0.10073066751162212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,128,0,1,fp8,fp8,0,0.11545600493748982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,128,0,1,float16,float16,0,0.1039626697699229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,128,0,1,float16,float16,0,0.10909866293271382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,128,0,1,float16,fp8,0,0.10760000348091125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,128,0,1,float16,float16,0,0.08016533156236012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,128,0,1,fp8,fp8,0,0.11874666810035706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,128,0,1,fp8,fp8,0,0.06300266583760579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,128,0,1,float16,fp8,0,0.07795733213424683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,128,0,1,fp8,fp8,0,0.07913066446781158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,128,0,1,float16,float16,0,0.055733333031336464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,128,0,1,float16,fp8,0,0.05669866502285004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,128,0,1,float16,float16,0,0.05649599929650625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,128,0,1,float16,fp8,0,0.056602666775385536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,128,0,1,fp8,fp8,0,0.06397866706053416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,128,0,1,float16,float16,0,0.05738133192062378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,128,0,1,float16,fp8,0,0.057349334160486855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,128,0,1,fp8,fp8,0,0.06554133196671803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,128,0,1,float16,float16,0,0.05975999931494395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,128,0,1,float16,fp8,0,0.05961066484451294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,128,0,1,fp8,fp8,0,0.06738666693369548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,128,0,1,float16,float16,0,0.03987200061480204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,128,0,1,float16,fp8,0,0.038618666430314384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,128,0,1,fp8,fp8,0,0.04570133487383524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,128,0,1,float16,float16,0,0.033861334125200905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,128,0,1,float16,fp8,0,0.034490667283535004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,128,0,1,fp8,fp8,0,0.03681066632270813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,128,0,1,fp8,fp8,0,0.0369759996732076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,128,0,1,float16,float16,0,0.03401066611210505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,128,0,1,float16,fp8,0,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,128,0,1,fp8,fp8,0,0.036501333117485046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,128,0,1,float16,float16,0,0.03433600068092346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,128,0,1,float16,fp8,0,0.03422400106986364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,128,0,1,float16,float16,0,0.034901333351929985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,128,0,1,float16,fp8,0,0.035045333206653595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,128,0,1,fp8,fp8,0,0.03830400109291077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,128,0,1,float16,float16,0,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,128,0,1,float16,fp8,0,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,128,0,1,fp8,fp8,0,0.026330667237440746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,128,0,1,fp8,fp8,0,0.024341332415739696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,128,0,1,float16,float16,0,0.022986667851607006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,128,0,1,float16,fp8,0,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,128,0,1,fp8,fp8,0,0.024069334069887798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,128,0,1,float16,float16,0,0.02330133318901062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,128,0,1,float16,fp8,0,0.02329600105683009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,128,0,1,float16,float16,0,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,128,0,1,float16,fp8,0,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,128,0,1,fp8,fp8,0,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,128,0,1,float16,float16,0,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,128,0,1,float16,fp8,0,0.02405333270629247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,128,0,1,fp8,fp8,0,0.025610665480295818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,128,0,1,float16,float16,0,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,128,0,1,float16,fp8,0,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,128,0,1,fp8,fp8,0,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,128,0,1,float16,float16,0,0.017722666263580322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,128,0,1,float16,fp8,0,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,128,0,1,fp8,fp8,0,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,128,0,1,float16,float16,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,128,0,1,float16,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,128,0,1,fp8,fp8,0,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,128,0,1,float16,float16,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,128,0,1,float16,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,128,0,1,fp8,fp8,0,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,128,0,1,float16,float16,0,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,128,0,1,float16,fp8,0,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,128,0,1,fp8,fp8,0,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,128,0,1,float16,float16,0,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,128,0,1,float16,fp8,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,128,0,1,fp8,fp8,0,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,128,0,1,float16,fp8,0,0.01591466615597407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,128,0,1,fp8,fp8,0,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,128,0,1,float16,float16,0,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,128,0,1,float16,fp8,0,0.015893333901961643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,128,0,1,fp8,fp8,0,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,128,0,1,float16,float16,0,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,128,0,1,float16,fp8,0,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,128,0,1,fp8,fp8,0,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,128,0,1,float16,fp8,0,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,128,0,1,float16,float16,0,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,128,0,1,float16,fp8,0,0.01616000011563301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,128,0,1,fp8,fp8,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,128,0,1,float16,float16,0,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,128,0,1,fp8,fp8,0,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,128,0,1,float16,float16,0,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,128,0,1,float16,fp8,0,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,128,0,1,fp8,fp8,0,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,128,0,1,float16,fp8,0,0.01573333392540614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,128,0,1,float16,float16,0,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,128,0,1,float16,fp8,0,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,128,0,1,fp8,fp8,0,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,128,0,1,float16,float16,0,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,128,0,1,fp8,fp8,0,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,128,0,1,float16,float16,0,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,128,0,1,float16,fp8,0,0.01584533353646596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,128,0,1,fp8,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,128,0,1,float16,float16,0,0.24995734294255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,128,0,1,float16,fp8,0,0.24916799863179526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,128,0,1,fp8,fp8,0,0.31441599130630493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,128,0,1,float16,float16,0,0.25300800800323486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,128,0,1,float16,float16,0,0.25727466742197674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,128,0,1,fp8,fp8,0,0.32039467493693036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,128,0,1,float16,fp8,0,0.25360000133514404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,128,0,1,fp8,fp8,0,0.319541335105896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,128,0,1,float16,float16,0,0.17529600858688354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,128,0,1,float16,fp8,0,0.2595040003458659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,128,0,1,float16,float16,0,0.2686079939206441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,128,0,1,float16,fp8,0,0.26562132438023883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,128,0,1,fp8,fp8,0,0.1926506757736206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,128,0,1,fp8,fp8,0,0.32781867186228436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,128,0,1,float16,float16,0,0.13154133160909018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,128,0,1,float16,fp8,0,0.17138133446375528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,128,0,1,float16,fp8,0,0.1322879989941915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,128,0,1,float16,float16,0,0.12814399600028992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,128,0,1,float16,fp8,0,0.12829867005348206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,128,0,1,fp8,fp8,0,0.16986666123072305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,128,0,1,fp8,fp8,0,0.1670080025990804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,128,0,1,fp8,fp8,0,0.16995733976364136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,128,0,1,float16,float16,0,0.13642133275667825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,128,0,1,float16,fp8,0,0.1360213359196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,128,0,1,float16,float16,0,0.14109866817792258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,128,0,1,float16,fp8,0,0.14075733224550882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,128,0,1,fp8,fp8,0,0.1741866668065389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,128,0,1,float16,float16,0,0.09386666615804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,128,0,1,float16,fp8,0,0.09158399701118469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,128,0,1,fp8,fp8,0,0.1062453289826711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,128,0,1,float16,float16,0,0.07039466500282288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,128,0,1,float16,fp8,0,0.07064533233642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,128,0,1,fp8,fp8,0,0.09004799524943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,128,0,1,float16,float16,0,0.07091199855009715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,128,0,1,float16,fp8,0,0.07131200035413106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,128,0,1,fp8,fp8,0,0.08972799777984619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,128,0,1,float16,float16,0,0.07233599821726482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,128,0,1,float16,float16,0,0.047413334250450134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,128,0,1,float16,fp8,0,0.07196799914042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,128,0,1,fp8,fp8,0,0.09171199798583984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,128,0,1,float16,float16,0,0.07493333518505096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,128,0,1,float16,fp8,0,0.04131199916203817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,128,0,1,float16,fp8,0,0.07387733459472656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,128,0,1,fp8,fp8,0,0.09547199805577596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,128,0,1,float16,fp8,0,0.04708800216515859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,128,0,1,fp8,fp8,0,0.060266668597857155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,128,0,1,float16,float16,0,0.04125866790612539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,128,0,1,fp8,fp8,0,0.05147199829419454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,128,0,1,fp8,fp8,0,0.05102399984995524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,128,0,1,float16,float16,0,0.04113066693147024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,128,0,1,float16,fp8,0,0.04081599911053976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,128,0,1,fp8,fp8,0,0.05120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,128,0,1,float16,float16,0,0.041482667128245033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,128,0,1,float16,fp8,0,0.04137066751718521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,128,0,1,float16,float16,0,0.041482667128245033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,128,0,1,float16,fp8,0,0.0415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,128,0,1,fp8,fp8,0,0.05234666665395101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,128,0,1,float16,float16,0,0.02880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,128,0,1,float16,fp8,0,0.027674667537212372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,128,0,1,fp8,fp8,0,0.03380800038576126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,128,0,1,float16,float16,0,0.026613332331180573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,128,0,1,float16,fp8,0,0.02609066665172577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,128,0,1,fp8,fp8,0,0.03172266731659571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,128,0,1,float16,float16,0,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,128,0,1,float16,float16,0,0.026682667434215546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,128,0,1,float16,fp8,0,0.02643733223279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,128,0,1,fp8,fp8,0,0.032069332897663116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,128,0,1,float16,float16,0,0.026677332818508148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,128,0,1,float16,fp8,0,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,128,0,1,fp8,fp8,0,0.03269333392381668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,128,0,1,float16,fp8,0,0.026506667335828144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,128,0,1,fp8,fp8,0,0.03272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,128,0,1,float16,float16,0,0.020037333170572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,128,0,1,float16,fp8,0,0.020031999796628952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,128,0,1,fp8,fp8,0,0.02370133250951767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,128,0,1,float16,float16,0,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,128,0,1,float16,fp8,0,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,128,0,1,fp8,fp8,0,0.021685334543387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,128,0,1,float16,float16,0,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,128,0,1,float16,fp8,0,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,128,0,1,float16,float16,0,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,128,0,1,float16,fp8,0,0.019402666638294857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,128,0,1,fp8,fp8,0,0.021829334398110706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,128,0,1,fp8,fp8,0,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,128,0,1,float16,float16,0,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,128,0,1,float16,float16,0,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,128,0,1,float16,fp8,0,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,128,0,1,fp8,fp8,0,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,128,0,1,float16,fp8,0,0.015909332782030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,128,0,1,fp8,fp8,0,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,128,0,1,float16,float16,0,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,128,0,1,float16,fp8,0,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,128,0,1,fp8,fp8,0,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,128,0,1,float16,float16,0,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,128,0,1,float16,fp8,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,128,0,1,fp8,fp8,0,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,128,0,1,float16,float16,0,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,128,0,1,float16,fp8,0,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,128,0,1,fp8,fp8,0,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,128,0,1,fp8,fp8,0,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,128,0,1,float16,fp8,0,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,128,0,1,fp8,fp8,0,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,128,0,1,float16,float16,0,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,128,0,1,float16,fp8,0,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,128,0,1,float16,float16,0,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,128,0,1,float16,fp8,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,128,0,1,fp8,fp8,0,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,128,0,1,float16,float16,0,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,128,0,1,float16,fp8,0,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,128,0,1,fp8,fp8,0,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,128,0,1,float16,float16,0,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,128,0,1,float16,fp8,0,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,128,0,1,float16,fp8,0,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,128,0,1,fp8,fp8,0,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,128,0,1,fp8,fp8,0,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,128,0,1,float16,float16,0,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,128,0,1,float16,fp8,0,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,128,0,1,float16,float16,0,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,128,0,1,fp8,fp8,0,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,128,0,1,float16,float16,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,128,0,1,float16,float16,0,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,128,0,1,float16,fp8,0,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,128,0,1,fp8,fp8,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,128,0,1,float16,fp8,0,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,128,0,1,fp8,fp8,0,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,128,0,1,float16,float16,0,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,128,0,1,float16,fp8,0,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,128,0,1,fp8,fp8,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,128,0,1,float16,float16,0,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,128,0,1,float16,fp8,0,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,128,0,1,fp8,fp8,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,128,0,1,float16,float16,0,0.20113599300384521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,128,0,1,float16,fp8,0,0.19955732425053915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,128,0,1,fp8,fp8,0,0.2767733335494995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,128,0,1,float16,float16,0,0.20388799905776978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,128,0,1,float16,fp8,0,0.2038559913635254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,128,0,1,fp8,fp8,0,0.27948800722757977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,128,0,1,float16,fp8,0,0.20782933632532755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,128,0,1,fp8,fp8,0,0.2802880009015401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,128,0,1,float16,float16,0,0.21478400627772012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,128,0,1,float16,float16,0,0.20849599440892538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,128,0,1,fp8,fp8,0,0.2832319935162862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,128,0,1,float16,fp8,0,0.21306133270263672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,128,0,1,float16,float16,0,0.12931733330090842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,128,0,1,float16,fp8,0,0.12742400169372559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,128,0,1,fp8,fp8,0,0.14574399590492249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,128,0,1,float16,float16,0,0.10643200079600017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,128,0,1,fp8,fp8,0,0.1597599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,128,0,1,fp8,fp8,0,0.14613333344459534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,128,0,1,float16,fp8,0,0.10635733604431152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,128,0,1,float16,float16,0,0.10593066612879436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,128,0,1,float16,fp8,0,0.105621337890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,128,0,1,float16,float16,0,0.10699733098347981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,128,0,1,float16,fp8,0,0.10708266496658325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,128,0,1,fp8,fp8,0,0.14730133612950644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,128,0,1,float16,float16,0,0.10929600397745769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,128,0,1,float16,fp8,0,0.10935466488202412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,128,0,1,float16,float16,0,0.06634133557478587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,128,0,1,fp8,fp8,0,0.15135467052459717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,128,0,1,float16,fp8,0,0.06541866560777028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,128,0,1,float16,fp8,0,0.05886933207511902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,128,0,1,fp8,fp8,0,0.0883893370628357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,128,0,1,float16,float16,0,0.058864002426465355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,128,0,1,float16,fp8,0,0.059119999408721924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,128,0,1,fp8,fp8,0,0.07877866427103679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,128,0,1,float16,float16,0,0.0588319996992747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,128,0,1,fp8,fp8,0,0.07912533481915791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,128,0,1,float16,fp8,0,0.05871466795603434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,128,0,1,float16,float16,0,0.059061333537101746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,128,0,1,fp8,fp8,0,0.0803466687599818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,128,0,1,float16,float16,0,0.05970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,128,0,1,float16,fp8,0,0.05926933387915293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,128,0,1,fp8,fp8,0,0.08063999811808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,128,0,1,float16,float16,0,0.036890665690104164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,128,0,1,float16,fp8,0,0.036890665690104164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,128,0,1,fp8,fp8,0,0.048512001832326256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,128,0,1,float16,float16,0,0.03563733398914337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,128,0,1,float16,fp8,0,0.03578133384386698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,128,0,1,fp8,fp8,0,0.04619733492533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,128,0,1,float16,float16,0,0.03548266738653183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,128,0,1,float16,fp8,0,0.03572266548871994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,128,0,1,fp8,fp8,0,0.04630400240421295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,128,0,1,float16,float16,0,0.03692800054947535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,128,0,1,float16,fp8,0,0.036501333117485046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,128,0,1,fp8,fp8,0,0.046896000703175865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,128,0,1,float16,float16,0,0.03603733330965042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,128,0,1,float16,fp8,0,0.036202666660149894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,128,0,1,float16,float16,0,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,128,0,1,fp8,fp8,0,0.04763199885686239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,128,0,1,float16,float16,0,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,128,0,1,float16,float16,0,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,128,0,1,float16,fp8,0,0.025455998877684276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,128,0,1,fp8,fp8,0,0.031130666534105938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,128,0,1,float16,fp8,0,0.02463999887307485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,128,0,1,fp8,fp8,0,0.029711998999118805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,128,0,1,float16,fp8,0,0.024495999018351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,128,0,1,float16,float16,0,0.024245334168275196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,128,0,1,fp8,fp8,0,0.029951999584833782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,128,0,1,float16,fp8,0,0.024746666351954143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,128,0,1,fp8,fp8,0,0.03018666555484136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,128,0,1,float16,float16,0,0.024469333390394848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,128,0,1,float16,fp8,0,0.024346667031447094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,128,0,1,fp8,fp8,0,0.030239999294281006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,128,0,1,float16,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,128,0,1,float16,float16,0,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,128,0,1,float16,fp8,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,128,0,1,fp8,fp8,0,0.022106667359670002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,128,0,1,float16,float16,0,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,128,0,1,fp8,fp8,0,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,128,0,1,float16,float16,0,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,128,0,1,float16,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,128,0,1,fp8,fp8,0,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,128,0,1,float16,float16,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,128,0,1,float16,fp8,0,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,128,0,1,fp8,fp8,0,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,128,0,1,float16,float16,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,128,0,1,float16,fp8,0,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,128,0,1,fp8,fp8,0,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,128,0,1,float16,float16,0,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,128,0,1,float16,fp8,0,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,128,0,1,fp8,fp8,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,128,0,1,float16,float16,0,0.014554666976133982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,128,0,1,fp8,fp8,0,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,128,0,1,float16,fp8,0,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,128,0,1,fp8,fp8,0,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,128,0,1,float16,float16,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,128,0,1,float16,fp8,0,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,128,0,1,float16,float16,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,128,0,1,float16,fp8,0,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,128,0,1,fp8,fp8,0,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,128,0,1,float16,float16,0,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,128,0,1,float16,fp8,0,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,128,0,1,fp8,fp8,0,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,128,0,1,float16,float16,0,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,128,0,1,float16,fp8,0,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,128,0,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,128,0,1,float16,float16,0,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,128,0,1,float16,fp8,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,128,0,1,fp8,fp8,0,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,128,0,1,fp8,fp8,0,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,128,0,1,float16,float16,0,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,128,0,1,fp8,fp8,0,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,128,0,1,float16,fp8,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,128,0,1,fp8,fp8,0,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,128,0,1,float16,float16,0,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,128,0,1,float16,fp8,0,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,128,0,1,float16,float16,0,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,128,0,1,float16,fp8,0,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,128,0,1,float16,float16,0,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,128,0,1,float16,fp8,0,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,128,0,1,fp8,fp8,0,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,128,0,1,float16,float16,0,0.013978666315476099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,128,0,1,float16,fp8,0,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,128,0,1,float16,fp8,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,128,0,1,fp8,fp8,0,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,128,0,1,float16,float16,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,128,0,1,float16,fp8,0,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,128,0,1,fp8,fp8,0,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,128,0,1,float16,float16,0,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,128,0,1,fp8,fp8,0,0.016341333587964375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,128,0,1,float16,float16,0,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,128,0,1,float16,fp8,0,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,128,0,1,fp8,fp8,0,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,128,0,1,fp8,fp8,0,16.366261800130207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,128,0,1,fp8,fp8,0,16.54429880777995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,128,0,1,float16,fp8,0,24.67101287841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,128,0,1,float16,float16,0,25.04602559407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,128,0,1,float16,float16,0,24.63689676920573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,128,0,1,float16,fp8,0,24.83472442626953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,128,0,1,float16,fp8,0,24.715354919433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,128,0,1,float16,float16,0,24.651568094889324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,128,0,1,fp8,fp8,0,18.30945078531901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,128,0,1,fp8,fp8,0,10.072805404663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,128,0,1,float16,float16,0,14.21240488688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,128,0,1,float16,fp8,0,14.434794108072916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,128,0,1,fp8,fp8,0,17.500357309977215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,128,0,1,float16,float16,0,26.821131388346355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,128,0,1,fp8,fp8,0,8.534554799397787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,128,0,1,float16,float16,0,12.389940897623697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,128,0,1,float16,fp8,0,12.570794423421225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,128,0,1,float16,fp8,0,25.361765543619793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,128,0,1,fp8,fp8,0,8.3733762105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,128,0,1,float16,float16,0,12.613919576009115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,128,0,1,float16,fp8,0,12.450939178466797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,128,0,1,fp8,fp8,0,8.36407470703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,128,0,1,float16,float16,0,12.625972747802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,128,0,1,float16,fp8,0,12.522074381510416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,128,0,1,fp8,fp8,0,4.379669189453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,128,0,1,float16,fp8,0,6.456042607625325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,128,0,1,float16,float16,0,6.54313596089681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,128,0,1,float16,float16,0,12.529706319173178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,128,0,1,fp8,fp8,0,8.631274541219076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,128,0,1,float16,float16,0,6.353066762288411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,128,0,1,fp8,fp8,0,4.473120053609212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,128,0,1,float16,fp8,0,6.391647974650065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,128,0,1,float16,fp8,0,12.963242848714193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,128,0,1,fp8,fp8,0,4.2845760981241865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,128,0,1,float16,float16,0,6.382501602172852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,128,0,1,float16,fp8,0,6.6640268961588545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,128,0,1,float16,float16,0,6.526159922281901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,128,0,1,fp8,fp8,0,4.813776016235352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,128,0,1,fp8,fp8,0,4.846320152282715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,128,0,1,float16,float16,0,3.9583520889282227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,128,0,1,float16,fp8,0,3.637455940246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,128,0,1,float16,fp8,0,7.274533589680989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,128,0,1,fp8,fp8,0,2.3900319735209146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,128,0,1,float16,float16,0,6.998741149902344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,128,0,1,float16,fp8,0,7.768208185831706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,128,0,1,float16,float16,0,3.392965316772461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,128,0,1,fp8,fp8,0,2.32204262415568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,128,0,1,fp8,fp8,0,2.380645275115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,128,0,1,float16,fp8,0,3.277477264404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,128,0,1,float16,float16,0,3.584805488586426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,128,0,1,float16,fp8,0,3.6382293701171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,128,0,1,float16,float16,0,3.427178700764974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,128,0,1,fp8,fp8,0,2.1556852658589682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,128,0,1,float16,fp8,0,3.571610768636068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,128,0,1,fp8,fp8,0,2.311749299367269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,128,0,1,float16,float16,0,3.435370763142904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,128,0,1,float16,fp8,0,3.568863868713379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,128,0,1,fp8,fp8,0,9.89678955078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,128,0,1,fp8,fp8,0,9.807114919026693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,128,0,1,float16,float16,0,14.5545654296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,128,0,1,float16,fp8,0,14.502843221028646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,128,0,1,float16,float16,0,14.622187296549479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,128,0,1,float16,fp8,0,14.629887898763021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,128,0,1,float16,float16,0,14.512208302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,128,0,1,float16,fp8,0,14.45583979288737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,128,0,1,fp8,fp8,0,9.808453241984049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,128,0,1,fp8,fp8,0,5.06821346282959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,128,0,1,float16,float16,0,7.633781433105469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,128,0,1,float16,fp8,0,7.751221338907878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,128,0,1,float16,float16,0,7.237701416015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,128,0,1,fp8,fp8,0,9.855914433797201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,128,0,1,float16,float16,0,15.23520533243815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,128,0,1,fp8,fp8,0,5.116853396097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,128,0,1,float16,fp8,0,7.585370381673177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,128,0,1,float16,fp8,0,15.567039489746094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,128,0,1,fp8,fp8,0,5.057600021362305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,128,0,1,float16,float16,0,7.868058522542317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,128,0,1,float16,fp8,0,8.640757242838541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,128,0,1,float16,float16,0,7.7113386789957685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,128,0,1,fp8,fp8,0,5.263285319010417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,128,0,1,float16,fp8,0,7.76035753885905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,128,0,1,float16,float16,0,7.778448104858398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,128,0,1,float16,float16,0,3.8842026392618814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,128,0,1,fp8,fp8,0,5.307034810384114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,128,0,1,fp8,fp8,0,2.791914621988932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,128,0,1,float16,fp8,0,3.978997230529785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,128,0,1,float16,float16,0,3.7926079432169595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,128,0,1,float16,fp8,0,7.532037099202474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,128,0,1,fp8,fp8,0,2.4665279388427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,128,0,1,float16,fp8,0,3.8238932291666665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,128,0,1,fp8,fp8,0,2.478874683380127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,128,0,1,float16,float16,0,3.6531041463216147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,128,0,1,float16,fp8,0,3.8299573262532554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,128,0,1,fp8,fp8,0,2.462560017903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,128,0,1,float16,float16,0,3.9817492167154946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,128,0,1,float16,fp8,0,3.8115412394205728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,128,0,1,fp8,fp8,0,1.3102506796518962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,128,0,1,float16,float16,0,2.0010080337524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,128,0,1,fp8,fp8,0,2.64300807317098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,128,0,1,float16,float16,0,3.844069480895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,128,0,1,float16,fp8,0,1.9260106086730957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,128,0,1,float16,fp8,0,3.821455955505371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,128,0,1,float16,float16,0,1.898586591084798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,128,0,1,float16,fp8,0,1.9394879341125488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,128,0,1,fp8,fp8,0,1.2703039646148682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,128,0,1,fp8,fp8,0,1.396399974822998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,128,0,1,float16,float16,0,1.8899680773417156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,128,0,1,float16,fp8,0,1.9130560557047527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,128,0,1,float16,float16,0,1.9314826329549153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,128,0,1,float16,fp8,0,1.8378879229227703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,128,0,1,fp8,fp8,0,1.3091200192769368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,128,0,1,float16,float16,0,1.9072693188985188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,128,0,1,fp8,fp8,0,1.323306639989217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,128,0,1,float16,fp8,0,1.9148906071980794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,128,0,1,fp8,fp8,0,7.230885187784831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,128,0,1,fp8,fp8,0,6.9432799021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,128,0,1,float16,float16,0,10.30683708190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,128,0,1,float16,fp8,0,10.293034871419271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,128,0,1,float16,float16,0,10.328725179036459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,128,0,1,float16,fp8,0,10.270661036173502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,128,0,1,float16,float16,0,10.355487823486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,128,0,1,fp8,fp8,0,6.9912109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,128,0,1,fp8,fp8,0,3.833552042643229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,128,0,1,float16,float16,0,5.507968266805013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,128,0,1,float16,fp8,0,5.52507209777832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,128,0,1,fp8,fp8,0,7.071610768636067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,128,0,1,float16,fp8,0,10.423871994018555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,128,0,1,float16,float16,0,10.751744588216146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,128,0,1,fp8,fp8,0,3.651151974995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,128,0,1,float16,fp8,0,10.468853632609049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,128,0,1,float16,float16,0,5.392447789510091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,128,0,1,float16,fp8,0,5.447754542032878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,128,0,1,fp8,fp8,0,3.516127904256185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,128,0,1,float16,float16,0,5.702367782592773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,128,0,1,float16,fp8,0,5.51908810933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,128,0,1,float16,float16,0,5.415818532307942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,128,0,1,fp8,fp8,0,3.7122720082600913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,128,0,1,float16,fp8,0,5.546592076619466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,128,0,1,float16,float16,0,2.9022719065348306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,128,0,1,fp8,fp8,0,3.7191041310628257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,128,0,1,float16,fp8,0,2.7557172775268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,128,0,1,float16,float16,0,5.59065055847168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,128,0,1,float16,fp8,0,5.539226531982422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,128,0,1,fp8,fp8,0,1.8746879895528157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,128,0,1,fp8,fp8,0,1.7869493166605632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,128,0,1,float16,float16,0,2.5817386309305825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,128,0,1,float16,fp8,0,2.7083733876546225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,128,0,1,fp8,fp8,0,1.751354694366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,128,0,1,float16,float16,0,2.647242705027262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,128,0,1,float16,fp8,0,2.7648693720499673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,128,0,1,float16,float16,0,2.6448747316996255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,128,0,1,float16,fp8,0,2.75928529103597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,128,0,1,fp8,fp8,0,1.7291199366251628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,128,0,1,float16,float16,0,1.410864035288493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,128,0,1,fp8,fp8,0,1.7942506472269695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,128,0,1,float16,fp8,0,1.4680320421854656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,128,0,1,fp8,fp8,0,0.9938133557637533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,128,0,1,float16,float16,0,2.718746821085612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,128,0,1,float16,fp8,0,2.585338592529297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,128,0,1,float16,fp8,0,1.3240906397501628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,128,0,1,float16,float16,0,1.4951252937316895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,128,0,1,fp8,fp8,0,0.9390239715576172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,128,0,1,fp8,fp8,0,0.9434773127237955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,128,0,1,float16,float16,0,1.385306676228841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,128,0,1,float16,fp8,0,1.3615840276082356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,128,0,1,fp8,fp8,0,0.9844426314036051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,128,0,1,float16,fp8,0,1.3774666786193848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,128,0,1,float16,float16,0,1.3827199935913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,128,0,1,float16,float16,0,1.3257493178049724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,128,0,1,float16,fp8,0,1.3305546442667644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,128,0,1,fp8,fp8,0,0.9408213297526041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,128,0,1,fp8,fp8,0,9.145135879516602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,128,0,1,fp8,fp8,0,9.264399846394857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,128,0,1,float16,fp8,0,13.868976593017578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,128,0,1,float16,float16,0,13.583904266357422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,128,0,1,float16,float16,0,14.158069610595703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,128,0,1,float16,fp8,0,13.609232584635416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,128,0,1,float16,float16,0,13.587125142415365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,128,0,1,float16,fp8,0,13.438847859700521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,128,0,1,fp8,fp8,0,9.456815719604492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,128,0,1,fp8,fp8,0,4.967653274536133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,128,0,1,float16,float16,0,7.327770868937175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,128,0,1,float16,fp8,0,7.258623758951823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,128,0,1,float16,float16,0,7.023802439371745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,128,0,1,fp8,fp8,0,9.655306498209635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,128,0,1,float16,float16,0,13.86950429280599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,128,0,1,fp8,fp8,0,4.67577584584554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,128,0,1,float16,fp8,0,7.31495475769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,128,0,1,fp8,fp8,0,4.984885215759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,128,0,1,float16,fp8,0,14.408191680908203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,128,0,1,float16,float16,0,7.249285380045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,128,0,1,float16,fp8,0,7.273658752441406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,128,0,1,fp8,fp8,0,4.887653350830078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,128,0,1,float16,float16,0,7.254400253295898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,128,0,1,float16,fp8,0,7.275594711303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,128,0,1,float16,float16,0,3.8062826792399087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,128,0,1,fp8,fp8,0,2.590709368387858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,128,0,1,float16,fp8,0,3.7861226399739585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,128,0,1,fp8,fp8,0,4.926917394002278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,128,0,1,float16,float16,0,7.17857042948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,128,0,1,float16,fp8,0,7.406965255737305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,128,0,1,float16,float16,0,3.510752042134603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,128,0,1,float16,fp8,0,3.448549270629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,128,0,1,fp8,fp8,0,2.306623935699463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,128,0,1,fp8,fp8,0,2.3233493169148765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,128,0,1,float16,float16,0,3.513807932535807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,128,0,1,float16,fp8,0,3.477914810180664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,128,0,1,fp8,fp8,0,2.280720074971517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,128,0,1,float16,float16,0,3.413306554158529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,128,0,1,float16,fp8,0,3.4399731953938804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,128,0,1,float16,float16,0,1.8065120379130046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,128,0,1,float16,float16,0,3.6893014907836914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,128,0,1,fp8,fp8,0,2.362127939860026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,128,0,1,float16,fp8,0,1.7337600390116374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,128,0,1,float16,fp8,0,3.486570676167806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,128,0,1,fp8,fp8,0,1.2677173614501953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,128,0,1,float16,float16,0,1.7004799842834473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,128,0,1,fp8,fp8,0,1.2342720031738281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,128,0,1,float16,fp8,0,1.758911927541097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,128,0,1,fp8,fp8,0,1.2296960353851318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,128,0,1,float16,float16,0,1.6545599301656086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,128,0,1,float16,fp8,0,1.6687520345052083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,128,0,1,float16,float16,0,1.7168960571289062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,128,0,1,float16,fp8,0,1.6690452893575032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,128,0,1,fp8,fp8,0,1.2413653532663982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,128,0,1,fp8,fp8,0,1.2019039789835613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,128,0,1,float16,float16,0,1.7216213544209797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,128,0,1,float16,float16,0,0.9707466761271158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,128,0,1,float16,fp8,0,1.7186773618062336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,128,0,1,fp8,fp8,0,0.6388746500015259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,128,0,1,float16,fp8,0,0.9152586460113525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,128,0,1,float16,float16,0,0.9166026910146078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,128,0,1,float16,fp8,0,0.8884533246358236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,128,0,1,fp8,fp8,0,0.6286453406016032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,128,0,1,float16,float16,0,0.9014240105946859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,128,0,1,fp8,fp8,0,0.6002346674601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,128,0,1,float16,fp8,0,0.9094293117523193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,128,0,1,fp8,fp8,0,0.6017493406931559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,128,0,1,float16,float16,0,0.9120746453603109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,128,0,1,float16,fp8,0,0.9161386489868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,128,0,1,float16,float16,0,0.9030133088429769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,128,0,1,fp8,fp8,0,0.6185333331425985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,128,0,1,float16,fp8,0,0.9003787040710449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,128,0,1,fp8,fp8,0,5.6410878499348955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,128,0,1,fp8,fp8,0,5.5013173421223955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,128,0,1,float16,float16,0,7.9274241129557295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,128,0,1,float16,fp8,0,7.958234786987305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,128,0,1,float16,float16,0,7.893557230631511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,128,0,1,float16,float16,0,7.9585920969645185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,128,0,1,float16,fp8,0,7.997541427612305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,128,0,1,float16,fp8,0,7.86949348449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,128,0,1,fp8,fp8,0,5.7515309651692705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,128,0,1,fp8,fp8,0,2.935663859049479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,128,0,1,float16,float16,0,4.187994639078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,128,0,1,float16,float16,0,4.06440003712972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,128,0,1,float16,fp8,0,4.230197270711263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,128,0,1,fp8,fp8,0,5.5855356852213545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,128,0,1,float16,float16,0,8.10748291015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,128,0,1,fp8,fp8,0,2.771120071411133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,128,0,1,float16,fp8,0,3.976282755533854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,128,0,1,float16,fp8,0,8.10414950052897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,128,0,1,fp8,fp8,0,2.7336212793986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,128,0,1,float16,float16,0,4.035317420959473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,128,0,1,float16,fp8,0,3.9558080037434897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,128,0,1,float16,float16,0,4.155520121256511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,128,0,1,fp8,fp8,0,2.877034823099772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,128,0,1,float16,fp8,0,4.138698577880859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,128,0,1,fp8,fp8,0,2.90336004892985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,128,0,1,float16,float16,0,2.1060372988382974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,128,0,1,float16,fp8,0,2.042170683542887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,128,0,1,fp8,fp8,0,1.4969760576883953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,128,0,1,float16,fp8,0,4.290863990783691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,128,0,1,float16,float16,0,4.154282569885254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,128,0,1,float16,float16,0,2.0613013903299966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,128,0,1,fp8,fp8,0,1.4405280749003093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,128,0,1,float16,fp8,0,2.0151732762654624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,128,0,1,float16,float16,0,2.04639466603597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,128,0,1,float16,fp8,0,1.9407413800557454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,128,0,1,fp8,fp8,0,1.3971306482950847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,128,0,1,fp8,fp8,0,1.3974186579386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,128,0,1,float16,float16,0,1.9617172876993816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,128,0,1,float16,fp8,0,1.967301368713379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,128,0,1,float16,float16,0,1.0387146472930908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,128,0,1,float16,float16,0,2.1080053647359214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,128,0,1,float16,fp8,0,1.0398026307423909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,128,0,1,float16,fp8,0,2.0883466402689614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,128,0,1,fp8,fp8,0,1.4460479418436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,128,0,1,fp8,fp8,0,0.7684853076934814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,128,0,1,float16,float16,0,1.0081173578898113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,128,0,1,float16,fp8,0,0.9940799872080485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,128,0,1,fp8,fp8,0,0.7158666451772054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,128,0,1,fp8,fp8,0,0.7864267031351725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,128,0,1,float16,float16,0,1.019274632136027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,128,0,1,float16,fp8,0,0.997221310933431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,128,0,1,float16,fp8,0,1.0102399984995525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,128,0,1,float16,float16,0,1.0159306526184082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,128,0,1,fp8,fp8,0,0.7433919906616211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,128,0,1,float16,float16,0,0.5749866565068563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,128,0,1,fp8,fp8,0,0.7597333590189616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,128,0,1,fp8,fp8,0,0.4296213388442993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,128,0,1,float16,float16,0,1.0065120061238606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,128,0,1,float16,fp8,0,0.5888053178787231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,128,0,1,float16,fp8,0,1.0098400115966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,128,0,1,float16,float16,0,0.5594720045725504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,128,0,1,float16,fp8,0,0.5491946538289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,128,0,1,fp8,fp8,0,0.3874560197194417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,128,0,1,float16,float16,0,0.5574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,128,0,1,fp8,fp8,0,0.38450666268666583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,128,0,1,float16,fp8,0,0.5760853290557861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,128,0,1,float16,float16,0,0.5536213318506876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,128,0,1,fp8,fp8,0,0.4002133210500081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,128,0,1,float16,fp8,0,0.5594826539357504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,128,0,1,float16,float16,0,0.5523200035095215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,128,0,1,fp8,fp8,0,0.38974400361378986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,128,0,1,float16,fp8,0,0.5619519948959351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,128,0,1,fp8,fp8,0,5.668218612670898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,128,0,1,fp8,fp8,0,5.479103724161784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,128,0,1,float16,fp8,0,7.6932799021403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,128,0,1,float16,float16,0,8.059605280558268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,128,0,1,float16,float16,0,7.934874852498372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,128,0,1,float16,fp8,0,7.847706476847331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,128,0,1,float16,float16,0,8.054704030354818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,128,0,1,fp8,fp8,0,5.415711720784505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,128,0,1,fp8,fp8,0,2.979215939839681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,128,0,1,float16,float16,0,4.146064122517903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,128,0,1,float16,fp8,0,4.129941304524739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,128,0,1,float16,fp8,0,8.064565022786459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,128,0,1,fp8,fp8,0,5.65506108601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,128,0,1,float16,float16,0,7.868176142374675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,128,0,1,fp8,fp8,0,2.6952266693115234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,128,0,1,float16,float16,0,3.966602643330892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,128,0,1,float16,fp8,0,7.91653315226237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,128,0,1,float16,fp8,0,4.005189259847005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,128,0,1,float16,float16,0,3.8868532180786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,128,0,1,fp8,fp8,0,2.715749422709147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,128,0,1,float16,fp8,0,3.6095733642578125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,128,0,1,float16,float16,0,3.8419628143310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,128,0,1,fp8,fp8,0,2.6492746671040854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,128,0,1,float16,float16,0,2.087839921315511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,128,0,1,float16,fp8,0,4.022794723510742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,128,0,1,fp8,fp8,0,2.818309466044108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,128,0,1,float16,fp8,0,2.087402661641439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,128,0,1,float16,float16,0,3.9355573654174805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,128,0,1,float16,fp8,0,4.105936050415039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,128,0,1,fp8,fp8,0,1.518954594930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,128,0,1,float16,float16,0,1.8146826426188152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,128,0,1,fp8,fp8,0,1.3258346716562908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,128,0,1,float16,fp8,0,1.8611520131429036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,128,0,1,fp8,fp8,0,1.3939894040425618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,128,0,1,float16,float16,0,1.8754933675130208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,128,0,1,float16,fp8,0,1.8637065887451172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,128,0,1,float16,float16,0,1.888485272725423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,128,0,1,fp8,fp8,0,1.3909653027852376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,128,0,1,float16,fp8,0,1.8510400454203289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,128,0,1,float16,float16,0,1.0149226983388264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,128,0,1,float16,float16,0,1.9183146158854167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,128,0,1,fp8,fp8,0,0.7836106618245443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,128,0,1,float16,fp8,0,1.0149813493092854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,128,0,1,fp8,fp8,0,1.4032853444417317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,128,0,1,float16,fp8,0,1.9622400601704915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,128,0,1,float16,float16,0,0.9349439938863119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,128,0,1,float16,fp8,0,0.9577120145161947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,128,0,1,fp8,fp8,0,0.6885759830474854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,128,0,1,float16,float16,0,0.9520533084869385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,128,0,1,fp8,fp8,0,0.7024266719818115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,128,0,1,float16,fp8,0,0.9383412996927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,128,0,1,float16,float16,0,0.9379519621531168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,128,0,1,fp8,fp8,0,0.7043146292368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,128,0,1,float16,fp8,0,0.9613760312398275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,128,0,1,fp8,fp8,0,0.7363466421763102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,128,0,1,float16,float16,0,0.956816037495931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,128,0,1,float16,fp8,0,0.9530826409657797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,128,0,1,float16,float16,0,0.5337973435719808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,128,0,1,fp8,fp8,0,0.4038506746292114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,128,0,1,float16,fp8,0,0.5371466477711996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,128,0,1,float16,float16,0,0.5045973459879557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,128,0,1,fp8,fp8,0,0.3739680051803589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,128,0,1,float16,fp8,0,0.5109173456827799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,128,0,1,float16,float16,0,0.5092586676279703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,128,0,1,float16,float16,0,0.5065386692682902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,128,0,1,fp8,fp8,0,0.3660373290379842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,128,0,1,float16,fp8,0,0.5078346729278564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,128,0,1,float16,fp8,0,0.5127626657485962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,128,0,1,fp8,fp8,0,0.36823999881744385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,128,0,1,float16,float16,0,0.512656013170878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,128,0,1,float16,fp8,0,0.5200800100962321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,128,0,1,fp8,fp8,0,0.3765706618626912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,128,0,1,float16,float16,0,0.27569599946339923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,128,0,1,float16,fp8,0,0.28012800216674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,128,0,1,fp8,fp8,0,0.22589333852132162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,128,0,1,float16,float16,0,0.2634986639022827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,128,0,1,float16,fp8,0,0.26518932978312176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,128,0,1,fp8,fp8,0,0.2098133365313212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,128,0,1,float16,float16,0,0.26471465826034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,128,0,1,float16,fp8,0,0.26572267214457196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,128,0,1,fp8,fp8,0,0.2103253404299418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,128,0,1,float16,float16,0,0.2649386723836263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,128,0,1,float16,fp8,0,0.2674880027770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,128,0,1,fp8,fp8,0,0.21185600757598877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,128,0,1,float16,float16,0,0.2680000066757202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,128,0,1,float16,fp8,0,0.2688746651013692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,128,0,1,fp8,fp8,0,0.21512534221013388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,128,0,1,fp8,fp8,0,3.34773317972819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,128,0,1,fp8,fp8,0,3.3656158447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,128,0,1,float16,float16,0,4.746389389038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,128,0,1,float16,fp8,0,4.745893478393555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,128,0,1,float16,float16,0,4.761546770731608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,128,0,1,float16,fp8,0,4.762965202331543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,128,0,1,float16,float16,0,4.801877339680989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,128,0,1,float16,fp8,0,4.839136123657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,128,0,1,fp8,fp8,0,3.450394630432129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,128,0,1,fp8,fp8,0,1.8857812881469727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,128,0,1,float16,float16,0,2.5129812558492026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,128,0,1,float16,fp8,0,2.490784009297689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,128,0,1,fp8,fp8,0,3.4302345911661782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,128,0,1,float16,float16,0,4.816416104634603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,128,0,1,float16,fp8,0,4.7825972239176435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,128,0,1,fp8,fp8,0,1.662293275197347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,128,0,1,float16,float16,0,2.270474592844645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,128,0,1,float16,fp8,0,2.262181282043457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,128,0,1,float16,float16,0,2.282069365183512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,128,0,1,float16,fp8,0,2.320458730061849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,128,0,1,fp8,fp8,0,1.7168052991231282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,128,0,1,float16,float16,0,2.320511976877848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,128,0,1,fp8,fp8,0,1.6663146018981934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,128,0,1,float16,fp8,0,2.3245226542154946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,128,0,1,float16,float16,0,1.23581329981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,128,0,1,float16,float16,0,2.4041120211283364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,128,0,1,float16,fp8,0,2.3270187377929688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,128,0,1,fp8,fp8,0,0.9552213350931803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,128,0,1,fp8,fp8,0,1.7627092997233074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,128,0,1,float16,float16,0,1.091370662053426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,128,0,1,float16,fp8,0,1.2207307020823162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,128,0,1,float16,fp8,0,1.1427253087361653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,128,0,1,fp8,fp8,0,0.8586773077646891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,128,0,1,float16,float16,0,1.1459733645121257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,128,0,1,fp8,fp8,0,0.8346613248189291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,128,0,1,float16,fp8,0,1.1225600242614746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,128,0,1,fp8,fp8,0,0.8717226982116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,128,0,1,float16,float16,0,1.110106627146403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,128,0,1,float16,fp8,0,1.1128693421681721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,128,0,1,float16,float16,0,1.1369493007659912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,128,0,1,float16,float16,0,0.6372533241907755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,128,0,1,fp8,fp8,0,0.507482647895813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,128,0,1,float16,fp8,0,0.6463040113449097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,128,0,1,float16,fp8,0,1.1362613042195637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,128,0,1,fp8,fp8,0,0.8875573476155599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,128,0,1,float16,float16,0,0.5734399954477946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,128,0,1,fp8,fp8,0,0.4522879918416341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,128,0,1,float16,fp8,0,0.5969280004501343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,128,0,1,float16,float16,0,0.5825599829355875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,128,0,1,float16,fp8,0,0.5911733309427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,128,0,1,fp8,fp8,0,0.45878398418426514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,128,0,1,float16,float16,0,0.5961973269780477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,128,0,1,fp8,fp8,0,0.46536000569661456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,128,0,1,float16,fp8,0,0.5973759889602661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,128,0,1,float16,float16,0,0.3456639846165975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,128,0,1,fp8,fp8,0,0.44948267936706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,128,0,1,float16,float16,0,0.5955520073572794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,128,0,1,float16,fp8,0,0.5974239905675253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,128,0,1,float16,fp8,0,0.34835199515024823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,128,0,1,fp8,fp8,0,0.2679786682128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,128,0,1,float16,float16,0,0.324890673160553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,128,0,1,float16,fp8,0,0.3266666730244954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,128,0,1,fp8,fp8,0,0.238319993019104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,128,0,1,float16,float16,0,0.320303996404012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,128,0,1,float16,fp8,0,0.32739200194676715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,128,0,1,fp8,fp8,0,0.23867199818293253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,128,0,1,float16,float16,0,0.32813332478205365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,128,0,1,float16,float16,0,0.1832586725552877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,128,0,1,fp8,fp8,0,0.24095465739568075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,128,0,1,float16,fp8,0,0.330677330493927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,128,0,1,float16,fp8,0,0.185263991355896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,128,0,1,float16,float16,0,0.3325279951095581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,128,0,1,fp8,fp8,0,0.2409493327140808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,128,0,1,float16,fp8,0,0.33204267422358197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,128,0,1,fp8,fp8,0,0.1528426706790924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,128,0,1,float16,float16,0,0.1741066575050354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,128,0,1,float16,fp8,0,0.1735573410987854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,128,0,1,fp8,fp8,0,0.14082666238149008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,128,0,1,float16,float16,0,0.17332800229390463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,128,0,1,float16,fp8,0,0.1757813294728597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,128,0,1,fp8,fp8,0,0.13873599966367087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,128,0,1,float16,float16,0,0.1744800011316935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,128,0,1,float16,fp8,0,0.17599467436472574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,128,0,1,fp8,fp8,0,0.14248533050219217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,128,0,1,float16,float16,0,0.17442667484283447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,128,0,1,float16,fp8,0,0.1774346629778544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,128,0,1,fp8,fp8,0,0.14563199877738953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,128,0,1,fp8,fp8,0,3.537109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,128,0,1,fp8,fp8,0,3.6059999465942383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,128,0,1,float16,float16,0,4.887413342793782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,128,0,1,float16,fp8,0,4.834021250406901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,128,0,1,float16,float16,0,4.848906517028809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,128,0,1,float16,fp8,0,4.843642552693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,128,0,1,float16,float16,0,4.852944056193034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,128,0,1,float16,fp8,0,4.8248640696207685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,128,0,1,fp8,fp8,0,3.6344321568806968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,128,0,1,float16,float16,0,2.5902560551961265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,128,0,1,fp8,fp8,0,2.0789333979288735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,128,0,1,float16,float16,0,2.2823947270711265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,128,0,1,float16,fp8,0,2.630074659983317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,128,0,1,fp8,fp8,0,3.703770637512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,128,0,1,float16,float16,0,5.042064030965169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,128,0,1,float16,fp8,0,4.940661430358887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,128,0,1,float16,fp8,0,2.290442625681559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,128,0,1,fp8,fp8,0,1.8234400749206543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,128,0,1,fp8,fp8,0,1.8034133911132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,128,0,1,float16,float16,0,2.330176035563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,128,0,1,float16,fp8,0,2.3946293195088706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,128,0,1,float16,float16,0,2.3501599629720054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,128,0,1,fp8,fp8,0,1.881600062052409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,128,0,1,float16,fp8,0,2.4033493995666504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,128,0,1,float16,float16,0,1.3194879690806072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,128,0,1,float16,float16,0,2.448597272237142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,128,0,1,fp8,fp8,0,1.8669974009195964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,128,0,1,float16,fp8,0,2.4287840525309243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,128,0,1,fp8,fp8,0,1.050602674484253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,128,0,1,float16,fp8,0,1.1250293254852295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,128,0,1,float16,float16,0,1.1299200057983398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,128,0,1,float16,fp8,0,1.2894826730092366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,128,0,1,fp8,fp8,0,0.9244266351064047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,128,0,1,float16,float16,0,1.1181386311848958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,128,0,1,float16,fp8,0,1.142416000366211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,128,0,1,fp8,fp8,0,0.905349334081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,128,0,1,float16,float16,0,1.1301653385162354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,128,0,1,fp8,fp8,0,0.9126506646474203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,128,0,1,float16,fp8,0,1.1512266794840496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,128,0,1,float16,float16,0,1.1564959685007732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,128,0,1,float16,float16,0,0.6629600127538046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,128,0,1,fp8,fp8,0,0.9340373675028483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,128,0,1,float16,fp8,0,0.6612799962361654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,128,0,1,float16,fp8,0,1.16156800587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,128,0,1,float16,float16,0,0.5821760098139445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,128,0,1,fp8,fp8,0,0.5393120050430298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,128,0,1,float16,fp8,0,0.5832906564076742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,128,0,1,fp8,fp8,0,0.4702026844024658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,128,0,1,float16,float16,0,0.5806453227996826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,128,0,1,float16,fp8,0,0.5820053418477377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,128,0,1,fp8,fp8,0,0.4732960065205892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,128,0,1,float16,float16,0,0.5915573438008627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,128,0,1,fp8,fp8,0,0.47861866156260174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,128,0,1,float16,fp8,0,0.5888213316599528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,128,0,1,float16,float16,0,0.5957119862238566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,128,0,1,float16,float16,0,0.3519573211669922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,128,0,1,float16,fp8,0,0.3543413480122884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,128,0,1,fp8,fp8,0,0.48737601439158124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,128,0,1,float16,fp8,0,0.5999253193537394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,128,0,1,float16,float16,0,0.3150773247083028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,128,0,1,float16,fp8,0,0.31470932563145954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,128,0,1,fp8,fp8,0,0.2902773420015971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,128,0,1,fp8,fp8,0,0.2444960077603658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,128,0,1,float16,float16,0,0.31481067339579266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,128,0,1,float16,float16,0,0.3184000054995219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,128,0,1,float16,fp8,0,0.3207626740137736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,128,0,1,float16,fp8,0,0.320415993531545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,128,0,1,fp8,fp8,0,0.24451200167338052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,128,0,1,fp8,fp8,0,0.2476053237915039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,128,0,1,float16,float16,0,0.321125328540802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,128,0,1,float16,fp8,0,0.32528533538182575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,128,0,1,float16,float16,0,0.1888213356335958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,128,0,1,float16,fp8,0,0.18862400452295938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,128,0,1,fp8,fp8,0,0.2547893325487773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,128,0,1,fp8,fp8,0,0.15706666310628256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,128,0,1,float16,float16,0,0.16693333784739176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,128,0,1,float16,fp8,0,0.16830400625864664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,128,0,1,fp8,fp8,0,0.13803733388582864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,128,0,1,float16,float16,0,0.16777066389719644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,128,0,1,float16,fp8,0,0.169429341952006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,128,0,1,fp8,fp8,0,0.13940800229708353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,128,0,1,float16,float16,0,0.16806934277216592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,128,0,1,float16,fp8,0,0.17069866259892783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,128,0,1,fp8,fp8,0,0.1404159963130951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,128,0,1,float16,float16,0,0.1715839902559916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,128,0,1,float16,fp8,0,0.1713599960009257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,128,0,1,fp8,fp8,0,0.1442400018374125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,128,0,1,float16,float16,0,0.10693867007891338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,128,0,1,float16,fp8,0,0.10730133454004924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,128,0,1,fp8,fp8,0,0.09430399537086487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,128,0,1,float16,float16,0,0.10086400310198466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,128,0,1,float16,fp8,0,0.1009333332379659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,128,0,1,fp8,fp8,0,0.08418132861455281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,128,0,1,float16,float16,0,0.09994133313496907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,128,0,1,float16,fp8,0,0.10121066371599834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,128,0,1,fp8,fp8,0,0.084389328956604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,128,0,1,float16,float16,0,0.10188266634941101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,128,0,1,float16,fp8,0,0.10187199711799622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,128,0,1,fp8,fp8,0,0.08641067147254944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,128,0,1,float16,float16,0,0.10223999619483948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,128,0,1,float16,fp8,0,0.1030346651871999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,128,0,1,fp8,fp8,0,0.08745066324869792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,128,0,1,fp8,fp8,0,2.389338652292887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,128,0,1,float16,float16,0,2.8932320276896157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,128,0,1,float16,fp8,0,2.9806931813557944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,128,0,1,float16,float16,0,2.9537121454874673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,128,0,1,fp8,fp8,0,2.388373374938965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,128,0,1,float16,fp8,0,2.9649171829223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,128,0,1,float16,float16,0,3.0203145345052085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,128,0,1,float16,fp8,0,3.0408480962117515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,128,0,1,fp8,fp8,0,2.4161814053853354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,128,0,1,float16,float16,0,1.6910559336344402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,128,0,1,float16,fp8,0,1.685215950012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,128,0,1,float16,float16,0,3.0930614471435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,128,0,1,fp8,fp8,0,1.420639991760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,128,0,1,float16,fp8,0,3.0890881220499673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,128,0,1,float16,float16,0,1.464410622914632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,128,0,1,fp8,fp8,0,2.484565258026123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,128,0,1,float16,fp8,0,1.4501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,128,0,1,fp8,fp8,0,1.2047680219014485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,128,0,1,float16,float16,0,1.4701439539591472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,128,0,1,fp8,fp8,0,1.1877333323160808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,128,0,1,float16,fp8,0,1.4578240712483723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,128,0,1,float16,float16,0,1.477392037709554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,128,0,1,fp8,fp8,0,1.2304960091908772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,128,0,1,float16,fp8,0,1.4530293146769206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,128,0,1,float16,float16,0,1.5147840181986492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,128,0,1,float16,float16,0,0.8545333544413248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,128,0,1,fp8,fp8,0,1.2656319936116536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,128,0,1,float16,fp8,0,0.8470026652018229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,128,0,1,float16,fp8,0,1.5276212692260742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,128,0,1,fp8,fp8,0,0.7214986483256022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,128,0,1,float16,float16,0,0.7181759675343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,128,0,1,fp8,fp8,0,0.6235520044962565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,128,0,1,float16,fp8,0,0.7344319820404053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,128,0,1,float16,fp8,0,0.7259199619293213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,128,0,1,float16,float16,0,0.7230453491210938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,128,0,1,fp8,fp8,0,0.6390346686045328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,128,0,1,float16,float16,0,0.7311306794484457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,128,0,1,fp8,fp8,0,0.6262933413187662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,128,0,1,float16,fp8,0,0.7349332968393961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,128,0,1,float16,float16,0,0.7498026688893636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,128,0,1,float16,float16,0,0.4467039903004964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,128,0,1,float16,fp8,0,0.7531786759694418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,128,0,1,fp8,fp8,0,0.6424266497294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,128,0,1,float16,fp8,0,0.44124801953633624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,128,0,1,fp8,fp8,0,0.37696532408396405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,128,0,1,float16,float16,0,0.37985066572825116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,128,0,1,float16,fp8,0,0.38040534655253094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,128,0,1,fp8,fp8,0,0.3240533272425334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,128,0,1,float16,float16,0,0.3824319839477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,128,0,1,fp8,fp8,0,0.3266879916191101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,128,0,1,float16,fp8,0,0.3854080041249593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,128,0,1,float16,float16,0,0.38621334234873456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,128,0,1,float16,fp8,0,0.388314684232076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,128,0,1,fp8,fp8,0,0.32848532994588214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,128,0,1,float16,float16,0,0.39363733927408856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,128,0,1,float16,fp8,0,0.3973653316497803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,128,0,1,float16,fp8,0,0.23862399657567343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,128,0,1,fp8,fp8,0,0.33774932225545246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,128,0,1,float16,float16,0,0.23896000782648721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,128,0,1,fp8,fp8,0,0.2000160018603007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,128,0,1,float16,float16,0,0.20918399095535278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,128,0,1,float16,fp8,0,0.2116960088411967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,128,0,1,fp8,fp8,0,0.16825066010157266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,128,0,1,float16,float16,0,0.21247466405232748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,128,0,1,float16,fp8,0,0.21266132593154907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,128,0,1,fp8,fp8,0,0.16915732622146606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,128,0,1,float16,float16,0,0.21246933937072754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,128,0,1,float16,fp8,0,0.2139306664466858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,128,0,1,fp8,fp8,0,0.17131733894348145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,128,0,1,float16,float16,0,0.21687465906143188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,128,0,1,float16,fp8,0,0.21985065937042236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,128,0,1,fp8,fp8,0,0.17536532878875732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,128,0,1,float16,float16,0,0.12876799702644348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,128,0,1,float16,fp8,0,0.13121066490809122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,128,0,1,fp8,fp8,0,0.11243733763694763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,128,0,1,float16,float16,0,0.1141866644223531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,128,0,1,float16,fp8,0,0.11571733156840007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,128,0,1,fp8,fp8,0,0.09795733292897542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,128,0,1,float16,float16,0,0.11507733662923177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,128,0,1,float16,fp8,0,0.11523200074831645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,128,0,1,fp8,fp8,0,0.0997973382472992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,128,0,1,fp8,fp8,0,0.10371200243631999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,128,0,1,float16,float16,0,0.11598400274912517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,128,0,1,float16,fp8,0,0.11723732948303223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,128,0,1,fp8,fp8,0,0.10070400436719258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,128,0,1,float16,fp8,0,0.07315200070540111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,128,0,1,float16,float16,0,0.1186133325099945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,128,0,1,fp8,fp8,0,0.06205333272616068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,128,0,1,float16,fp8,0,0.12012799580891927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,128,0,1,float16,float16,0,0.07520533104737599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,128,0,1,float16,fp8,0,0.07688533266385396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,128,0,1,fp8,fp8,0,0.07018666466077168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,128,0,1,float16,float16,0,0.07243200143178304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,128,0,1,float16,float16,0,0.07228800157705943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,128,0,1,float16,fp8,0,0.0729066679875056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,128,0,1,fp8,fp8,0,0.06182399888833364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,128,0,1,float16,float16,0,0.07250133156776428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,128,0,1,float16,fp8,0,0.07237866520881653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,128,0,1,fp8,fp8,0,0.061477333307266235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,128,0,1,float16,float16,0,0.07229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,128,0,1,float16,fp8,0,0.07378666599591573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,128,0,1,fp8,fp8,0,0.06407999992370605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,128,0,1,float16,float16,0,3.023791948954264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,128,0,1,fp8,fp8,0,2.63920005162557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,128,0,1,float16,fp8,0,2.9882027308146157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,128,0,1,fp8,fp8,0,2.6429386138916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,128,0,1,float16,float16,0,2.91702938079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,128,0,1,float16,fp8,0,3.030010541280111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,128,0,1,float16,float16,0,3.199381192525228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,128,0,1,float16,fp8,0,2.9501492182413735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,128,0,1,float16,float16,0,1.9476213455200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,128,0,1,float16,float16,0,1.4984532992045085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,128,0,1,float16,fp8,0,1.8699466387430828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,128,0,1,fp8,fp8,0,1.6198986371358235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,128,0,1,fp8,fp8,0,2.9171625773111978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,128,0,1,float16,float16,0,3.2885119120279946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,128,0,1,float16,fp8,0,3.1942666371663413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,128,0,1,fp8,fp8,0,2.928208033243815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,128,0,1,fp8,fp8,0,1.33843199412028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,128,0,1,float16,fp8,0,1.4880639712015789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,128,0,1,float16,float16,0,1.4824479420979817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,128,0,1,float16,fp8,0,1.4893706639607747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,128,0,1,fp8,fp8,0,1.3769386609395344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,128,0,1,float16,fp8,0,1.4977973302205403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,128,0,1,float16,float16,0,1.5292107264200847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,128,0,1,fp8,fp8,0,1.4781066576639812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,128,0,1,float16,float16,0,0.9711200396219889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,128,0,1,float16,float16,0,1.5816960334777832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,128,0,1,float16,fp8,0,1.5639467239379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,128,0,1,fp8,fp8,0,1.4899999300638835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,128,0,1,fp8,fp8,0,0.8237067063649496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,128,0,1,float16,fp8,0,0.944597323735555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,128,0,1,float16,float16,0,0.7576906681060791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,128,0,1,float16,fp8,0,0.7593226432800293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,128,0,1,fp8,fp8,0,0.669706662495931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,128,0,1,float16,float16,0,0.769599994023641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,128,0,1,float16,fp8,0,0.7595626513163248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,128,0,1,fp8,fp8,0,0.6694293022155762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,128,0,1,float16,float16,0,0.766266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,128,0,1,float16,fp8,0,0.7686613400777181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,128,0,1,fp8,fp8,0,0.751418670018514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,128,0,1,float16,float16,0,0.4939146836598714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,128,0,1,float16,float16,0,0.8042666912078857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,128,0,1,float16,fp8,0,0.801530679066976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,128,0,1,fp8,fp8,0,0.7535999615987142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,128,0,1,float16,fp8,0,0.47778133551279706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,128,0,1,fp8,fp8,0,0.4143413305282593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,128,0,1,float16,float16,0,0.3927839994430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,128,0,1,float16,fp8,0,0.38869333267211914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,128,0,1,fp8,fp8,0,0.3421226739883423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,128,0,1,float16,float16,0,0.39188798268636066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,128,0,1,float16,fp8,0,0.3943466742833455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,128,0,1,fp8,fp8,0,0.34229334195454914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,128,0,1,float16,float16,0,0.4020693302154541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,128,0,1,float16,fp8,0,0.3988746802012126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,128,0,1,fp8,fp8,0,0.3821973403294881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,128,0,1,float16,float16,0,0.41464531421661377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,128,0,1,float16,fp8,0,0.4121599992116292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,128,0,1,fp8,fp8,0,0.38602666060129803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,128,0,1,float16,float16,0,0.25945067405700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,128,0,1,float16,fp8,0,0.2534613410631816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,128,0,1,fp8,fp8,0,0.21640533208847046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,128,0,1,float16,float16,0,0.208079993724823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,128,0,1,float16,fp8,0,0.21042132377624512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,128,0,1,fp8,fp8,0,0.16331733266512552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,128,0,1,float16,float16,0,0.20919466018676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,128,0,1,float16,fp8,0,0.2091253399848938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,128,0,1,fp8,fp8,0,0.16587199767430624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,128,0,1,float16,float16,0,0.21462933222452799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,128,0,1,float16,fp8,0,0.21095999081929526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,128,0,1,fp8,fp8,0,0.18064000209172568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,128,0,1,float16,float16,0,0.22100265820821127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,128,0,1,float16,fp8,0,0.21893866856892905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,128,0,1,fp8,fp8,0,0.1815626621246338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,128,0,1,float16,float16,0,0.15079466501871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,128,0,1,fp8,fp8,0,0.11162133018175761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,128,0,1,float16,float16,0,0.1102186640103658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,128,0,1,float16,fp8,0,0.11169067025184631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,128,0,1,float16,fp8,0,0.1467519998550415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,128,0,1,float16,fp8,0,0.11044800281524658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,128,0,1,float16,float16,0,0.11646933356920879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,128,0,1,fp8,fp8,0,0.09070932865142822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,128,0,1,float16,float16,0,0.11250133315722148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,128,0,1,fp8,fp8,0,0.09153067072232564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,128,0,1,float16,fp8,0,0.11495467027028401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,128,0,1,float16,fp8,0,0.07163733243942261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,128,0,1,fp8,fp8,0,0.09737066427866618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,128,0,1,float16,float16,0,0.11628266175587972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,128,0,1,fp8,fp8,0,0.09819733103116353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,128,0,1,float16,fp8,0,0.1156213382879893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,128,0,1,float16,float16,0,0.07357866565386455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,128,0,1,fp8,fp8,0,0.06374933322270711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,128,0,1,float16,float16,0,0.060133333007494606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,128,0,1,float16,fp8,0,0.06039999922116598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,128,0,1,fp8,fp8,0,0.05052266518274943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,128,0,1,float16,float16,0,0.06073066592216492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,128,0,1,float16,fp8,0,0.0603359987338384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,128,0,1,fp8,fp8,0,0.051354666550954185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,128,0,1,float16,float16,0,0.063360000650088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,128,0,1,float16,fp8,0,0.062458669145902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,128,0,1,fp8,fp8,0,0.05482133229573568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,128,0,1,float16,float16,0,0.06451733410358429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,128,0,1,float16,fp8,0,0.06431999802589417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,128,0,1,fp8,fp8,0,0.05672533313433329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,128,0,1,float16,float16,0,0.03998400022586187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,128,0,1,float16,fp8,0,0.039264000952243805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,128,0,1,fp8,fp8,0,0.03620799879233042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,128,0,1,float16,float16,0,0.03611200054486593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,128,0,1,float16,fp8,0,0.0364533339937528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,128,0,1,fp8,fp8,0,0.03201066702604294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,128,0,1,float16,float16,0,0.03658133248488108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,128,0,1,float16,fp8,0,0.03708266715208689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,128,0,1,fp8,fp8,0,0.03279466678698858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,128,0,1,float16,float16,0,0.037615999579429626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,128,0,1,float16,fp8,0,0.03799466788768768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,128,0,1,fp8,fp8,0,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,128,0,1,float16,float16,0,0.03755199909210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,128,0,1,float16,fp8,0,0.03753600021203359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,128,0,1,fp8,fp8,0,0.033887999753157295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,128,0,1,float16,float16,0,2.2933600743611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,128,0,1,float16,fp8,0,2.2739893595377603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,128,0,1,fp8,fp8,0,2.144495964050293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,128,0,1,float16,float16,0,2.309226671854655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,128,0,1,fp8,fp8,0,2.174287954966227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,128,0,1,float16,fp8,0,2.3068745930989585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,128,0,1,float16,float16,0,2.39790932337443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,128,0,1,float16,float16,0,1.6089919408162434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,128,0,1,float16,fp8,0,2.3216212590535483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,128,0,1,float16,fp8,0,1.55239470799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,128,0,1,fp8,fp8,0,2.3858933448791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,128,0,1,float16,float16,0,2.507930596669515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,128,0,1,float16,fp8,0,2.4489493370056152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,128,0,1,fp8,fp8,0,2.45031468073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,128,0,1,float16,float16,0,1.1545013586680095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,128,0,1,float16,fp8,0,1.1651306947072346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,128,0,1,fp8,fp8,0,1.0817759831746419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,128,0,1,fp8,fp8,0,1.3729759852091472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,128,0,1,float16,float16,0,1.1702666282653809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,128,0,1,float16,fp8,0,1.1685760021209717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,128,0,1,fp8,fp8,0,1.0914613405863445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,128,0,1,float16,float16,0,1.1881759961446126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,128,0,1,float16,fp8,0,1.1852373282114665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,128,0,1,float16,float16,0,0.8142613569895426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,128,0,1,fp8,fp8,0,1.2196586926778157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,128,0,1,float16,fp8,0,0.7837653160095215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,128,0,1,fp8,fp8,0,0.6867733001708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,128,0,1,float16,float16,0,1.2579519748687744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,128,0,1,float16,fp8,0,1.2434240182240803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,128,0,1,fp8,fp8,0,1.240063985188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,128,0,1,float16,float16,0,0.5949173370997111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,128,0,1,fp8,fp8,0,0.5421386559804281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,128,0,1,float16,fp8,0,0.5922346512476603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,128,0,1,float16,float16,0,0.5964639981587728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,128,0,1,float16,fp8,0,0.5858293374379476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,128,0,1,fp8,fp8,0,0.5465493202209473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,128,0,1,float16,float16,0,0.605733315149943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,128,0,1,float16,fp8,0,0.604853351910909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,128,0,1,fp8,fp8,0,0.6206666628519694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,128,0,1,float16,float16,0,0.6465493440628052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,128,0,1,float16,float16,0,0.41228266557057697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,128,0,1,float16,fp8,0,0.39851733048756915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,128,0,1,float16,fp8,0,0.6366879940032959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,128,0,1,fp8,fp8,0,0.626858671506246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,128,0,1,fp8,fp8,0,0.3460906744003296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,128,0,1,float16,float16,0,0.3064799904823303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,128,0,1,float16,fp8,0,0.306442658106486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,128,0,1,fp8,fp8,0,0.2791573405265808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,128,0,1,float16,float16,0,0.3044106761614482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,128,0,1,float16,fp8,0,0.3069760004679362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,128,0,1,fp8,fp8,0,0.2821706732114156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,128,0,1,float16,float16,0,0.3167946736017863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,128,0,1,float16,fp8,0,0.3277600010236104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,128,0,1,float16,fp8,0,0.3118026653925578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,128,0,1,fp8,fp8,0,0.3168586691220601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,128,0,1,float16,float16,0,0.33481065432230633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,128,0,1,float16,float16,0,0.21719467639923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,128,0,1,fp8,fp8,0,0.31387199958165485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,128,0,1,float16,fp8,0,0.21019200483957926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,128,0,1,float16,float16,0,0.16544000307718912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,128,0,1,fp8,fp8,0,0.1850773294766744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,128,0,1,float16,float16,0,0.16220266620318094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,128,0,1,float16,fp8,0,0.16207999984423319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,128,0,1,fp8,fp8,0,0.1357599993546804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,128,0,1,float16,fp8,0,0.16371200482050577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,128,0,1,fp8,fp8,0,0.1366986632347107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,128,0,1,float16,fp8,0,0.16815465688705444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,128,0,1,float16,float16,0,0.169429341952006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,128,0,1,fp8,fp8,0,0.15054399768511453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,128,0,1,fp8,fp8,0,0.09717333316802979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,128,0,1,float16,float16,0,0.17492266496022543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,128,0,1,float16,fp8,0,0.1739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,128,0,1,fp8,fp8,0,0.15384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,128,0,1,float16,float16,0,0.13289599617322287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,128,0,1,float16,fp8,0,0.09053333600362141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,128,0,1,fp8,fp8,0,0.07719466586907704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,128,0,1,float16,fp8,0,0.12897066275278726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,128,0,1,float16,float16,0,0.08957333366076152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,128,0,1,float16,fp8,0,0.08973333239555359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,128,0,1,fp8,fp8,0,0.07639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,128,0,1,float16,float16,0,0.09133332967758179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,128,0,1,float16,float16,0,0.09438932935396831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,128,0,1,float16,fp8,0,0.09425066908200581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,128,0,1,fp8,fp8,0,0.08277866741021474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,128,0,1,float16,float16,0,0.09533333778381348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,128,0,1,float16,fp8,0,0.09449066718419392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,128,0,1,fp8,fp8,0,0.08291733264923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,128,0,1,float16,float16,0,0.06421866516272227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,128,0,1,float16,fp8,0,0.06233599781990051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,128,0,1,fp8,fp8,0,0.05685866872469584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,128,0,1,float16,float16,0,0.050527999798456825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,128,0,1,float16,fp8,0,0.050842667619387306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,128,0,1,fp8,fp8,0,0.04331199824810028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,128,0,1,float16,float16,0,0.05087999999523163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,128,0,1,float16,fp8,0,0.050981332858403526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,128,0,1,fp8,fp8,0,0.04340266684691111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,128,0,1,float16,float16,0,0.05285866558551788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,128,0,1,float16,fp8,0,0.051967998345692955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,128,0,1,fp8,fp8,0,0.04830400149027506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,128,0,1,float16,float16,0,0.05486399928728739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,128,0,1,float16,fp8,0,0.054229333996772766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,128,0,1,fp8,fp8,0,0.04916800061861674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,128,0,1,float16,fp8,0,0.032218667368094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,128,0,1,fp8,fp8,0,0.02882666637500127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,128,0,1,float16,float16,0,0.0365280012289683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,128,0,1,float16,fp8,0,0.03259733319282532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,128,0,1,float16,fp8,0,0.03492266684770584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,128,0,1,fp8,fp8,0,0.0324799989660581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,128,0,1,float16,float16,0,0.03218133250872294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,128,0,1,float16,float16,0,0.03215466688076655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,128,0,1,float16,float16,0,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,128,0,1,fp8,fp8,0,0.02923733244339625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,128,0,1,float16,float16,0,0.03332799921433131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,128,0,1,float16,fp8,0,0.033013333876927696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,128,0,1,fp8,fp8,0,0.030837332208951313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,128,0,1,float16,fp8,0,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,128,0,1,fp8,fp8,0,0.030975999931494396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,128,0,1,float16,float16,0,0.026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,128,0,1,float16,fp8,0,0.026533332963784535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,128,0,1,fp8,fp8,0,0.02459733436505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,128,0,1,float16,float16,0,0.024879999458789825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,128,0,1,float16,fp8,0,0.025087999800841015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,128,0,1,float16,float16,0,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,128,0,1,fp8,fp8,0,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,128,0,1,fp8,fp8,0,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,128,0,1,float16,float16,0,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,128,0,1,float16,fp8,0,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,128,0,1,fp8,fp8,0,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,128,0,1,float16,fp8,0,0.026021334032217663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,128,0,1,float16,float16,0,0.026613332331180573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,128,0,1,float16,fp8,0,0.02701333413521449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,128,0,1,fp8,fp8,0,0.02422400067249934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,128,0,1,float16,float16,0,0.9970400333404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,128,0,1,float16,fp8,0,0.9928692976633707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,128,0,1,fp8,fp8,0,0.8711626529693604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,128,0,1,float16,float16,0,1.0055306752522786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,128,0,1,float16,fp8,0,1.0014453728993733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,128,0,1,fp8,fp8,0,0.8945919672648112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,128,0,1,float16,float16,0,1.0506986776987712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,128,0,1,float16,fp8,0,1.0220426718393962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,128,0,1,fp8,fp8,0,0.9964426358540853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,128,0,1,float16,float16,0,1.1065119902292888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,128,0,1,float16,float16,0,0.7440746625264486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,128,0,1,float16,fp8,0,1.0856106281280518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,128,0,1,fp8,fp8,0,1.0340159734090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,128,0,1,float16,fp8,0,0.7115200360616049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,128,0,1,float16,float16,0,0.5067893266677856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,128,0,1,fp8,fp8,0,0.5957386493682861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,128,0,1,float16,fp8,0,0.5064693291982015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,128,0,1,fp8,fp8,0,0.43485867977142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,128,0,1,float16,float16,0,0.5132586558659872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,128,0,1,float16,fp8,0,0.5110133488972982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,128,0,1,float16,float16,0,0.5268319845199585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,128,0,1,float16,fp8,0,0.5223946571350098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,128,0,1,fp8,fp8,0,0.4434986511866252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,128,0,1,fp8,fp8,0,0.5077653328577677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,128,0,1,float16,float16,0,0.5661866664886475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,128,0,1,float16,fp8,0,0.5533653497695923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,128,0,1,float16,float16,0,0.38089601198832196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,128,0,1,float16,fp8,0,0.26333866516749066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,128,0,1,fp8,fp8,0,0.3014880021413167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,128,0,1,float16,fp8,0,0.3633813460667928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,128,0,1,float16,fp8,0,0.26603732506434125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,128,0,1,fp8,fp8,0,0.5172160069147745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,128,0,1,float16,float16,0,0.26364266872406006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,128,0,1,fp8,fp8,0,0.2254133423169454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,128,0,1,fp8,fp8,0,0.2640533248583476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,128,0,1,float16,float16,0,0.2653333346048991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,128,0,1,fp8,fp8,0,0.22817599773406982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,128,0,1,float16,float16,0,0.2750506599744161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,128,0,1,float16,fp8,0,0.27380265792210895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,128,0,1,float16,float16,0,0.2938773234685262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,128,0,1,float16,fp8,0,0.28777599334716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,128,0,1,fp8,fp8,0,0.26444266239802044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,128,0,1,float16,float16,0,0.20160533984502158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,128,0,1,float16,fp8,0,0.1943626602490743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,128,0,1,fp8,fp8,0,0.16995733976364136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,128,0,1,float16,float16,0,0.14043733477592468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,128,0,1,float16,fp8,0,0.14241066575050354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,128,0,1,fp8,fp8,0,0.1202239990234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,128,0,1,float16,float16,0,0.14306666453679404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,128,0,1,float16,fp8,0,0.14308266838391623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,128,0,1,fp8,fp8,0,0.12381333112716675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,128,0,1,float16,float16,0,0.1476959983507792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,128,0,1,float16,fp8,0,0.14793066183725992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,128,0,1,fp8,fp8,0,0.13648000359535217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,128,0,1,float16,float16,0,0.15660267074902853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,128,0,1,float16,fp8,0,0.15520000457763672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,128,0,1,fp8,fp8,0,0.138373335202535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,128,0,1,fp8,fp8,0,0.06911999980608623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,128,0,1,float16,float16,0,0.12077333529790242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,128,0,1,float16,fp8,0,0.11719466249148051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,128,0,1,fp8,fp8,0,0.09117333094278972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,128,0,1,float16,fp8,0,0.08507733543713887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,128,0,1,float16,float16,0,0.08026133477687836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,128,0,1,float16,fp8,0,0.08096000055472057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,128,0,1,float16,float16,0,0.08182399968306224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,128,0,1,float16,fp8,0,0.0817333310842514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,128,0,1,fp8,fp8,0,0.07005866865317027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,128,0,1,float16,float16,0,0.0858026643594106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,128,0,1,fp8,fp8,0,0.07553066809972127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,128,0,1,float16,float16,0,0.08881066242853801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,128,0,1,float16,fp8,0,0.08689066767692566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,128,0,1,fp8,fp8,0,0.07561600208282471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,128,0,1,float16,float16,0,0.060975998640060425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,128,0,1,float16,fp8,0,0.058634668588638306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,128,0,1,fp8,fp8,0,0.05276266733805338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,128,0,1,float16,float16,0,0.04562133550643921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,128,0,1,float16,fp8,0,0.04549333453178406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,128,0,1,fp8,fp8,0,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,128,0,1,float16,float16,0,0.04674666623274485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,128,0,1,float16,fp8,0,0.045850664377212524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,128,0,1,fp8,fp8,0,0.03944533318281174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,128,0,1,float16,fp8,0,0.049466664592425026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,128,0,1,fp8,fp8,0,0.045312002301216125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,128,0,1,float16,float16,0,0.04888000090916952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,128,0,1,float16,fp8,0,0.047983999053637184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,128,0,1,fp8,fp8,0,0.0439626673857371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,128,0,1,float16,float16,0,0.049440001447995506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,128,0,1,float16,float16,0,0.034314667185147606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,128,0,1,float16,float16,0,0.03148799886306127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,128,0,1,float16,fp8,0,0.032816000282764435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,128,0,1,fp8,fp8,0,0.03121600051720937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,128,0,1,float16,float16,0,0.03014933317899704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,128,0,1,float16,fp8,0,0.03068800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,128,0,1,fp8,fp8,0,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,128,0,1,float16,fp8,0,0.031040000418821972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,128,0,1,fp8,fp8,0,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,128,0,1,fp8,fp8,0,0.02956266701221466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,128,0,1,float16,float16,0,0.03187733391920725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,128,0,1,float16,fp8,0,0.03205333401759466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,128,0,1,fp8,fp8,0,0.028666667640209198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,128,0,1,float16,float16,0,0.03162133445342382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,128,0,1,float16,fp8,0,0.03187733391920725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,128,0,1,float16,float16,0,0.02390933285156886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,128,0,1,float16,fp8,0,0.024186665813128155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,128,0,1,fp8,fp8,0,0.022634667654832203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,128,0,1,float16,float16,0,0.022810667753219604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,128,0,1,float16,fp8,0,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,128,0,1,fp8,fp8,0,0.020096000283956528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,128,0,1,float16,float16,0,0.02254933367172877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,128,0,1,float16,fp8,0,0.022533332308133442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,128,0,1,fp8,fp8,0,0.020693333198626835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,128,0,1,float16,float16,0,0.023552000522613525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,128,0,1,float16,fp8,0,0.023546665906906128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,128,0,1,fp8,fp8,0,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,128,0,1,float16,float16,0,0.02372266600529353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,128,0,1,float16,fp8,0,0.023317334552605946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,128,0,1,fp8,fp8,0,0.02186666677395503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,128,0,1,float16,float16,0,0.019632000476121902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,128,0,1,float16,fp8,0,0.019893333315849304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,128,0,1,fp8,fp8,0,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,128,0,1,float16,float16,0,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,128,0,1,float16,fp8,0,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,128,0,1,fp8,fp8,0,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,128,0,1,fp8,fp8,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,128,0,1,float16,float16,0,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,128,0,1,float16,fp8,0,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,128,0,1,fp8,fp8,0,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,128,0,1,float16,float16,0,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,128,0,1,float16,fp8,0,0.019567999988794327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,128,0,1,float16,float16,0,0.019567999988794327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,128,0,1,float16,fp8,0,0.01970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,128,0,1,fp8,fp8,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,128,0,1,float16,float16,0,0.4734453360239665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,128,0,1,float16,fp8,0,0.4732053279876709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,128,0,1,fp8,fp8,0,0.4122133255004883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,128,0,1,float16,float16,0,0.4811573425928752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,128,0,1,fp8,fp8,0,0.4421439965565999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,128,0,1,float16,fp8,0,0.47785067558288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,128,0,1,float16,float16,0,0.49670934677124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,128,0,1,float16,fp8,0,0.4913119872411092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,128,0,1,fp8,fp8,0,0.5015360116958618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,128,0,1,float16,float16,0,0.532975991566976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,128,0,1,float16,float16,0,0.38938132921854657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,128,0,1,float16,fp8,0,0.5244959990183512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,128,0,1,fp8,fp8,0,0.5242240031560262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,128,0,1,float16,float16,0,0.2490826646486918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,128,0,1,fp8,fp8,0,0.3052266637484233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,128,0,1,float16,fp8,0,0.3733280102411906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,128,0,1,float16,fp8,0,0.2483946681022644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,128,0,1,fp8,fp8,0,0.21754666169484457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,128,0,1,float16,float16,0,0.2518506646156311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,128,0,1,float16,fp8,0,0.2508959968884786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,128,0,1,fp8,fp8,0,0.22256000836690268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,128,0,1,float16,float16,0,0.2656960090001424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,128,0,1,float16,fp8,0,0.2611680030822754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,128,0,1,fp8,fp8,0,0.26447467009226483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,128,0,1,fp8,fp8,0,0.25933865706125897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,128,0,1,fp8,fp8,0,0.157669335603714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,128,0,1,float16,float16,0,0.2816106677055359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,128,0,1,float16,fp8,0,0.27847466866175336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,128,0,1,float16,float16,0,0.20681599775950113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,128,0,1,float16,fp8,0,0.19829867283503214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,128,0,1,float16,float16,0,0.13529599706331888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,128,0,1,float16,fp8,0,0.1360319952170054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,128,0,1,fp8,fp8,0,0.11581866939862569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,128,0,1,float16,float16,0,0.13819733262062073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,128,0,1,float16,fp8,0,0.1373546620210012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,128,0,1,fp8,fp8,0,0.13354667027791342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,128,0,1,fp8,fp8,0,0.12035733461380005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,128,0,1,float16,float16,0,0.14357333381970724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,128,0,1,float16,fp8,0,0.14321600397427878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,128,0,1,float16,float16,0,0.15237866838773093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,128,0,1,float16,fp8,0,0.15436800320943198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,128,0,1,fp8,fp8,0,0.13611732920010886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,128,0,1,float16,float16,0,0.11155200004577637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,128,0,1,float16,fp8,0,0.10843732953071594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,128,0,1,fp8,fp8,0,0.06639466683069865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,128,0,1,float16,float16,0,0.07598400115966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,128,0,1,fp8,fp8,0,0.08759466807047527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,128,0,1,fp8,fp8,0,0.06888533135255177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,128,0,1,float16,float16,0,0.07335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,128,0,1,float16,fp8,0,0.07406933108965556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,128,0,1,float16,fp8,0,0.07548800110816956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,128,0,1,float16,float16,0,0.08159466584523518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,128,0,1,float16,fp8,0,0.07970666885375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,128,0,1,fp8,fp8,0,0.07413333157698314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,128,0,1,float16,fp8,0,0.05787200232346853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,128,0,1,float16,float16,0,0.08543466528256734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,128,0,1,float16,fp8,0,0.08362666765848796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,128,0,1,fp8,fp8,0,0.07566933333873749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,128,0,1,float16,float16,0,0.059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,128,0,1,fp8,fp8,0,0.05222400029500326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,128,0,1,float16,float16,0,0.043136000633239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,128,0,1,float16,fp8,0,0.043178667624791466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,128,0,1,fp8,fp8,0,0.038106667498747505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,128,0,1,float16,float16,0,0.04390933116277059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,128,0,1,float16,fp8,0,0.04363733530044556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,128,0,1,fp8,fp8,0,0.03863999992609024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,128,0,1,float16,float16,0,0.046629334489504494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,128,0,1,float16,fp8,0,0.04595200220743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,128,0,1,float16,float16,0,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,128,0,1,fp8,fp8,0,0.04312533140182495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,128,0,1,float16,float16,0,0.04701866706212362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,128,0,1,float16,fp8,0,0.04630400240421295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,128,0,1,fp8,fp8,0,0.04438933233420054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,128,0,1,float16,fp8,0,0.03339733431736628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,128,0,1,fp8,fp8,0,0.03126933425664902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,128,0,1,float16,float16,0,0.030581332743167877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,128,0,1,fp8,fp8,0,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,128,0,1,float16,fp8,0,0.031328000128269196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,128,0,1,fp8,fp8,0,0.02718399961789449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,128,0,1,float16,float16,0,0.031119999786218006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,128,0,1,float16,fp8,0,0.03107200066248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,128,0,1,float16,float16,0,0.023311999936898548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,128,0,1,float16,float16,0,0.03200000027815501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,128,0,1,float16,fp8,0,0.03201066702604294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,128,0,1,fp8,fp8,0,0.028794666131337483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,128,0,1,float16,float16,0,0.03183466692765554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,128,0,1,float16,fp8,0,0.031370667119820915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,128,0,1,fp8,fp8,0,0.02979733298222224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,128,0,1,float16,fp8,0,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,128,0,1,fp8,fp8,0,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,128,0,1,float16,float16,0,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,128,0,1,float16,fp8,0,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,128,0,1,fp8,fp8,0,0.020128000527620316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,128,0,1,float16,float16,0,0.022837333381175995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,128,0,1,float16,fp8,0,0.021727999051411945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,128,0,1,fp8,fp8,0,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,128,0,1,float16,float16,0,0.022128000855445862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,128,0,1,float16,fp8,0,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,128,0,1,fp8,fp8,0,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,128,0,1,float16,fp8,0,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,128,0,1,fp8,fp8,0,0.022053333620230358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,128,0,1,float16,float16,0,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,128,0,1,float16,fp8,0,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,128,0,1,fp8,fp8,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,128,0,1,float16,float16,0,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,128,0,1,float16,fp8,0,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,128,0,1,fp8,fp8,0,0.01781333362062772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,128,0,1,fp8,fp8,0,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,128,0,1,float16,float16,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,128,0,1,float16,fp8,0,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,128,0,1,fp8,fp8,0,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,128,0,1,float16,float16,0,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,128,0,1,float16,fp8,0,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,128,0,1,float16,float16,0,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,128,0,1,float16,fp8,0,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,128,0,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,128,0,1,float16,float16,0,0.016682667036851246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,128,0,1,float16,fp8,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,128,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,128,0,1,float16,float16,0,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,128,0,1,float16,fp8,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,128,0,1,fp8,fp8,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,128,0,1,float16,float16,0,0.016613333175579708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,128,0,1,float16,fp8,0,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,128,0,1,fp8,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,128,0,1,float16,float16,0,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,128,0,1,float16,fp8,0,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,128,0,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,128,0,1,float16,float16,0,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,128,0,1,float16,fp8,0,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,128,0,1,fp8,fp8,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,128,0,1,float16,float16,0,0.27880533536275226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,128,0,1,float16,fp8,0,0.27909332513809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,128,0,1,fp8,fp8,0,0.2994346618652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,128,0,1,float16,float16,0,0.2823786735534668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,128,0,1,float16,fp8,0,0.2801919976870219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,128,0,1,fp8,fp8,0,0.31886933247248334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,128,0,1,float16,float16,0,0.2877119978268941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,128,0,1,float16,fp8,0,0.2855466604232788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,128,0,1,fp8,fp8,0,0.3449813524881999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,128,0,1,float16,float16,0,0.3097760081291199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,128,0,1,float16,fp8,0,0.30110933383305866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,128,0,1,float16,float16,0,0.20523200432459512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,128,0,1,float16,fp8,0,0.19847999016443887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,128,0,1,fp8,fp8,0,0.3510293165842692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,128,0,1,fp8,fp8,0,0.19756799936294556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,128,0,1,float16,float16,0,0.1474560002485911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,128,0,1,float16,fp8,0,0.14758400122324625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,128,0,1,fp8,fp8,0,0.15958399573961893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,128,0,1,float16,float16,0,0.14819733301798502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,128,0,1,float16,fp8,0,0.14844800035158792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,128,0,1,fp8,fp8,0,0.1632960041364034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,128,0,1,float16,float16,0,0.1529973347981771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,128,0,1,float16,fp8,0,0.15179733435312906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,128,0,1,fp8,fp8,0,0.17575999101003012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,128,0,1,float16,float16,0,0.16145066420237222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,128,0,1,float16,fp8,0,0.15989866852760315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,128,0,1,fp8,fp8,0,0.1786080002784729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,128,0,1,float16,float16,0,0.11029866337776184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,128,0,1,float16,fp8,0,0.10596266388893127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,128,0,1,fp8,fp8,0,0.10878933469454448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,128,0,1,float16,float16,0,0.07782933115959167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,128,0,1,float16,fp8,0,0.07804266611735027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,128,0,1,fp8,fp8,0,0.08829333384831746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,128,0,1,fp8,fp8,0,0.09648000200589497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,128,0,1,float16,float16,0,0.07914133369922638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,128,0,1,float16,fp8,0,0.07961066563924153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,128,0,1,fp8,fp8,0,0.09063466389973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,128,0,1,float16,float16,0,0.08358400066693623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,128,0,1,float16,fp8,0,0.058320000767707825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,128,0,1,float16,fp8,0,0.0832479993502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,128,0,1,float16,float16,0,0.08574933807055156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,128,0,1,float16,fp8,0,0.08552533388137817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,128,0,1,fp8,fp8,0,0.0978559950987498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,128,0,1,float16,float16,0,0.05970133344332377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,128,0,1,fp8,fp8,0,0.061797335743904114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,128,0,1,float16,float16,0,0.046037331223487854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,128,0,1,float16,float16,0,0.048783997694651283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,128,0,1,float16,fp8,0,0.045066664616266884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,128,0,1,float16,float16,0,0.049551998575528465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,128,0,1,fp8,fp8,0,0.04956800242265066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,128,0,1,float16,float16,0,0.04569066564242045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,128,0,1,float16,fp8,0,0.045663997530937195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,128,0,1,fp8,fp8,0,0.05036800106366476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,128,0,1,float16,fp8,0,0.04796266555786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,128,0,1,fp8,fp8,0,0.05459199845790863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,128,0,1,float16,fp8,0,0.0487360010544459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,128,0,1,fp8,fp8,0,0.05507733424504598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,128,0,1,float16,float16,0,0.03199466566244761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,128,0,1,float16,fp8,0,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,128,0,1,fp8,fp8,0,0.03356799980004629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,128,0,1,float16,float16,0,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,128,0,1,float16,fp8,0,0.02903999884923299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,128,0,1,fp8,fp8,0,0.031125334401925404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,128,0,1,float16,float16,0,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,128,0,1,float16,fp8,0,0.02957333376010259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,128,0,1,fp8,fp8,0,0.031119999786218006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,128,0,1,float16,float16,0,0.030559999247392017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,128,0,1,float16,fp8,0,0.029946667452653248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,128,0,1,fp8,fp8,0,0.03236266722281774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,128,0,1,float16,float16,0,0.03067733347415924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,128,0,1,float16,fp8,0,0.02992533395687739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,128,0,1,fp8,fp8,0,0.0327360009153684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,128,0,1,float16,float16,0,0.02478400121132533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,128,0,1,float16,fp8,0,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,128,0,1,fp8,fp8,0,0.025850666066010792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,128,0,1,float16,float16,0,0.02385599911212921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,128,0,1,float16,float16,0,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,128,0,1,float16,fp8,0,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,128,0,1,fp8,fp8,0,0.02399466683467229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,128,0,1,float16,float16,0,0.023557332654794056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,128,0,1,float16,fp8,0,0.023733332753181458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,128,0,1,fp8,fp8,0,0.024266667664051056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,128,0,1,float16,fp8,0,0.02462399999300639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,128,0,1,fp8,fp8,0,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,128,0,1,float16,float16,0,0.02439466615517934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,128,0,1,float16,fp8,0,0.024480000138282776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,128,0,1,fp8,fp8,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,128,0,1,fp8,fp8,0,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,128,0,1,float16,float16,0,0.017743999759356182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,128,0,1,float16,fp8,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,128,0,1,fp8,fp8,0,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,128,0,1,float16,float16,0,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,128,0,1,float16,fp8,0,0.01681600014368693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,128,0,1,float16,float16,0,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,128,0,1,float16,float16,0,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,128,0,1,float16,fp8,0,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,128,0,1,fp8,fp8,0,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,128,0,1,float16,float16,0,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,128,0,1,float16,fp8,0,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,128,0,1,fp8,fp8,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,128,0,1,float16,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,128,0,1,fp8,fp8,0,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,128,0,1,float16,float16,0,0.01595199977358182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,128,0,1,float16,fp8,0,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,128,0,1,fp8,fp8,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,128,0,1,float16,fp8,0,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,128,0,1,float16,float16,0,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,128,0,1,float16,fp8,0,0.01607999950647354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,128,0,1,fp8,fp8,0,0.016336000214020412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,128,0,1,float16,float16,0,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,128,0,1,fp8,fp8,0,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,128,0,1,float16,float16,0,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,128,0,1,float16,fp8,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,128,0,1,float16,fp8,0,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,128,0,1,fp8,fp8,0,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,128,0,1,float16,float16,0,0.01594666639963786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,128,0,1,float16,fp8,0,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,128,0,1,fp8,fp8,0,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,128,0,1,float16,float16,0,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,128,0,1,fp8,fp8,0,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,128,0,1,float16,float16,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,128,0,1,float16,fp8,0,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,128,0,1,fp8,fp8,0,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,128,0,1,float16,float16,0,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,128,0,1,float16,fp8,0,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,128,0,1,fp8,fp8,0,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,128,0,1,float16,float16,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,128,0,1,float16,fp8,0,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,128,0,1,fp8,fp8,0,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,128,0,1,float16,float16,0,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,128,0,1,float16,fp8,0,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,128,0,1,fp8,fp8,0,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,128,0,1,float16,float16,0,0.19356799125671387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,128,0,1,float16,fp8,0,0.1931999921798706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,128,0,1,fp8,fp8,0,0.24046399195988974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,128,0,1,float16,float16,0,0.1946559945742289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,128,0,1,float16,fp8,0,0.19657599925994873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,128,0,1,fp8,fp8,0,0.24379199743270874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,128,0,1,float16,float16,0,0.1991680065790812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,128,0,1,float16,fp8,0,0.19892799854278564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,128,0,1,fp8,fp8,0,0.25759466489156085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,128,0,1,float16,float16,0,0.20977065960566202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,128,0,1,float16,fp8,0,0.20602667331695557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,128,0,1,float16,float16,0,0.13344533244768778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,128,0,1,fp8,fp8,0,0.26126400629679364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,128,0,1,float16,fp8,0,0.12940266728401184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,128,0,1,fp8,fp8,0,0.14921067158381143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,128,0,1,float16,float16,0,0.09990933537483215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,128,0,1,float16,fp8,0,0.1002346674601237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,128,0,1,fp8,fp8,0,0.12983999649683634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,128,0,1,float16,float16,0,0.10243733723958333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,128,0,1,float16,fp8,0,0.10293333729108174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,128,0,1,fp8,fp8,0,0.13176533579826355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,128,0,1,float16,float16,0,0.10592533151308696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,128,0,1,float16,fp8,0,0.10663466652234395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,128,0,1,fp8,fp8,0,0.13776000340779623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,128,0,1,float16,float16,0,0.10965333382288615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,128,0,1,float16,fp8,0,0.10960533221562703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,128,0,1,fp8,fp8,0,0.13807466626167297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,128,0,1,fp8,fp8,0,0.06984533369541168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,128,0,1,float16,fp8,0,0.05584533512592316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,128,0,1,float16,float16,0,0.07149333258469899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,128,0,1,float16,fp8,0,0.06991466879844666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,128,0,1,fp8,fp8,0,0.08284266789754231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,128,0,1,float16,float16,0,0.05610666672388712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,128,0,1,float16,fp8,0,0.05580266813437144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,128,0,1,float16,float16,0,0.05609600245952606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,128,0,1,fp8,fp8,0,0.07120533287525177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,128,0,1,float16,float16,0,0.0365226666132609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,128,0,1,float16,float16,0,0.05853866537412008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,128,0,1,float16,fp8,0,0.057717333237330117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,128,0,1,fp8,fp8,0,0.07650133470694225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,128,0,1,float16,float16,0,0.06051200131575266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,128,0,1,float16,fp8,0,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,128,0,1,fp8,fp8,0,0.0415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,128,0,1,float16,fp8,0,0.06027733286221822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,128,0,1,fp8,fp8,0,0.07650133470694225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,128,0,1,float16,fp8,0,0.03632533301909765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,128,0,1,fp8,fp8,0,0.04588800172011057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,128,0,1,float16,float16,0,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,128,0,1,float16,float16,0,0.03403733422358831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,128,0,1,float16,fp8,0,0.033786666889985405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,128,0,1,fp8,fp8,0,0.04204800228277842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,128,0,1,float16,float16,0,0.03389333436886469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,128,0,1,fp8,fp8,0,0.04363200068473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,128,0,1,float16,fp8,0,0.03500800083080927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,128,0,1,fp8,fp8,0,0.04354133208592733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,128,0,1,float16,float16,0,0.035071998834609985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,128,0,1,float16,fp8,0,0.03501333296298981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,128,0,1,float16,float16,0,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,128,0,1,float16,fp8,0,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,128,0,1,fp8,fp8,0,0.030640001098314922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,128,0,1,float16,float16,0,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,128,0,1,float16,fp8,0,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,128,0,1,fp8,fp8,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,128,0,1,float16,float16,0,0.023946667710940044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,128,0,1,float16,fp8,0,0.024266667664051056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,128,0,1,fp8,fp8,0,0.028490667541821797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,128,0,1,float16,float16,0,0.024314666787783306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,128,0,1,float16,fp8,0,0.024469333390394848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,128,0,1,fp8,fp8,0,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,128,0,1,float16,float16,0,0.02492266645034154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,128,0,1,float16,fp8,0,0.0249439999461174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,128,0,1,fp8,fp8,0,0.02951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,128,0,1,float16,float16,0,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,128,0,1,float16,fp8,0,0.019962667177120846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,128,0,1,fp8,fp8,0,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,128,0,1,float16,float16,0,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,128,0,1,float16,fp8,0,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,128,0,1,fp8,fp8,0,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,128,0,1,float16,float16,0,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,128,0,1,float16,fp8,0,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,128,0,1,fp8,fp8,0,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,128,0,1,float16,float16,0,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,128,0,1,fp8,fp8,0,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,128,0,1,float16,fp8,0,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,128,0,1,fp8,fp8,0,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,128,0,1,fp8,fp8,0,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,128,0,1,float16,float16,0,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,128,0,1,float16,fp8,0,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,128,0,1,float16,float16,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,128,0,1,float16,fp8,0,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,128,0,1,float16,float16,0,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,128,0,1,float16,fp8,0,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,128,0,1,fp8,fp8,0,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,128,0,1,float16,float16,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,128,0,1,float16,fp8,0,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,128,0,1,fp8,fp8,0,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,128,0,1,float16,float16,0,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,128,0,1,fp8,fp8,0,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,128,0,1,float16,fp8,0,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,128,0,1,fp8,fp8,0,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,128,0,1,float16,float16,0,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,128,0,1,float16,fp8,0,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,128,0,1,float16,fp8,0,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,128,0,1,float16,float16,0,0.014426667243242264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,128,0,1,float16,float16,0,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,128,0,1,float16,fp8,0,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,128,0,1,fp8,fp8,0,0.01670933390657107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,128,0,1,float16,float16,0,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,128,0,1,fp8,fp8,0,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,128,0,1,float16,fp8,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,128,0,1,fp8,fp8,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,128,0,1,float16,float16,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,128,0,1,float16,fp8,0,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,128,0,1,fp8,fp8,0,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,128,0,1,float16,fp8,0,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,128,0,1,float16,float16,0,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,128,0,1,float16,fp8,0,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,128,0,1,fp8,fp8,0,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,128,0,1,float16,float16,0,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,128,0,1,fp8,fp8,0,0.016058667252461117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,128,0,1,float16,float16,0,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,128,0,1,float16,fp8,0,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,128,0,1,float16,float16,0,0.013925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,128,0,1,fp8,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,128,0,1,fp8,fp8,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,128,0,1,float16,fp8,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,128,0,1,fp8,fp8,0,0.016202667107184727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,128,0,1,float16,float16,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,128,0,1,float16,fp8,0,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,128,0,1,float16,float16,0,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,128,0,1,float16,fp8,0,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,128,0,1,fp8,fp8,0,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,128,0,1,float16,float16,0,0.152346670627594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,128,0,1,float16,fp8,0,0.15426133076349893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,128,0,1,fp8,fp8,0,0.21356266736984253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,128,0,1,float16,float16,0,0.15466666221618652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,128,0,1,float16,fp8,0,0.15410133202870688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,128,0,1,float16,fp8,0,0.1632480025291443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,128,0,1,fp8,fp8,0,0.21471999088923135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,128,0,1,fp8,fp8,0,0.22129599253336588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,128,0,1,float16,float16,0,0.15778666734695435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,128,0,1,float16,fp8,0,0.15916267037391663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,128,0,1,fp8,fp8,0,0.2207146684328715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,128,0,1,float16,float16,0,0.16428800423940024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,128,0,1,float16,float16,0,0.08376000324885051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,128,0,1,float16,float16,0,0.0988159974416097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,128,0,1,float16,fp8,0,0.09571199615796407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,128,0,1,fp8,fp8,0,0.12617599964141846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,128,0,1,float16,float16,0,0.08284799754619598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,128,0,1,float16,fp8,0,0.08269333342711131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,128,0,1,fp8,fp8,0,0.11216533184051514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,128,0,1,float16,fp8,0,0.08345599969228108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,128,0,1,float16,float16,0,0.08710400263468425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,128,0,1,fp8,fp8,0,0.11340799927711487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,128,0,1,float16,float16,0,0.08597866694132487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,128,0,1,float16,fp8,0,0.08427199721336365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,128,0,1,fp8,fp8,0,0.11691733201344807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,128,0,1,float16,fp8,0,0.08644800384839375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,128,0,1,fp8,fp8,0,0.11777599652608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,128,0,1,fp8,fp8,0,0.06213866670926412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,128,0,1,float16,float16,0,0.050794666012128196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,128,0,1,float16,fp8,0,0.050901333491007485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,128,0,1,fp8,fp8,0,0.06676266590754192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,128,0,1,float16,float16,0,0.04744533201058706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,128,0,1,float16,fp8,0,0.047925333182017006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,128,0,1,float16,float16,0,0.04765866696834564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,128,0,1,float16,fp8,0,0.047781333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,128,0,1,fp8,fp8,0,0.06268266836802165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,128,0,1,float16,float16,0,0.048810665806134544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,128,0,1,float16,fp8,0,0.048751999934514366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,128,0,1,fp8,fp8,0,0.0647680014371872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,128,0,1,float16,float16,0,0.0487413356701533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,128,0,1,float16,fp8,0,0.04854933420817057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,128,0,1,fp8,fp8,0,0.06465599934260051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,128,0,1,float16,float16,0,0.032255999743938446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,128,0,1,float16,fp8,0,0.032474666833877563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,128,0,1,fp8,fp8,0,0.040549332896868386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,128,0,1,float16,float16,0,0.0308746670683225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,128,0,1,float16,fp8,0,0.03033600002527237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,128,0,1,fp8,fp8,0,0.03899200012286504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,128,0,1,float16,float16,0,0.03107200066248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,128,0,1,float16,fp8,0,0.031013332307338715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,128,0,1,fp8,fp8,0,0.03924266745646795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,128,0,1,float16,float16,0,0.031530665854612984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,128,0,1,float16,fp8,0,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,128,0,1,fp8,fp8,0,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,128,0,1,float16,float16,0,0.03156800071398417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,128,0,1,float16,fp8,0,0.03172266731659571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,128,0,1,fp8,fp8,0,0.04032533367474874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,128,0,1,float16,float16,0,0.0215786670645078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,128,0,1,float16,fp8,0,0.02223466585079829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,128,0,1,fp8,fp8,0,0.02629866699377696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,128,0,1,float16,float16,0,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,128,0,1,float16,fp8,0,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,128,0,1,fp8,fp8,0,0.024874667326609295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,128,0,1,float16,float16,0,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,128,0,1,float16,fp8,0,0.021520001192887623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,128,0,1,fp8,fp8,0,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,128,0,1,float16,float16,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,128,0,1,float16,fp8,0,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,128,0,1,fp8,fp8,0,0.0258240004380544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,128,0,1,float16,float16,0,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,128,0,1,float16,fp8,0,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,128,0,1,fp8,fp8,0,0.025861332813898723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,128,0,1,float16,float16,0,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,128,0,1,float16,fp8,0,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,128,0,1,fp8,fp8,0,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,128,0,1,float16,float16,0,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,128,0,1,float16,fp8,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,128,0,1,fp8,fp8,0,0.020736000190178554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,128,0,1,float16,float16,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,128,0,1,float16,fp8,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,128,0,1,fp8,fp8,0,0.020810666183630627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,128,0,1,float16,float16,0,0.017770666629076004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,128,0,1,float16,fp8,0,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,128,0,1,fp8,fp8,0,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,128,0,1,float16,float16,0,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,128,0,1,float16,fp8,0,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,128,0,1,fp8,fp8,0,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,128,0,1,float16,float16,0,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,128,0,1,float16,fp8,0,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,128,0,1,fp8,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,128,0,1,float16,float16,0,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,128,0,1,float16,fp8,0,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,128,0,1,fp8,fp8,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,128,0,1,float16,float16,0,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,128,0,1,float16,fp8,0,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,128,0,1,fp8,fp8,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,128,0,1,float16,float16,0,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,128,0,1,float16,fp8,0,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,128,0,1,fp8,fp8,0,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,128,0,1,float16,float16,0,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,128,0,1,float16,fp8,0,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,128,0,1,fp8,fp8,0,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,128,0,1,float16,float16,0,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,128,0,1,float16,fp8,0,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,128,0,1,fp8,fp8,0,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,128,0,1,float16,float16,0,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,128,0,1,float16,fp8,0,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,128,0,1,fp8,fp8,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,128,0,1,float16,float16,0,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,128,0,1,float16,fp8,0,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,128,0,1,fp8,fp8,0,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,128,0,1,float16,float16,0,0.014389333625634512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,128,0,1,float16,fp8,0,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,128,0,1,fp8,fp8,0,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,128,0,1,float16,float16,0,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,128,0,1,float16,fp8,0,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,128,0,1,fp8,fp8,0,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,128,0,1,float16,float16,0,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,128,0,1,float16,fp8,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,128,0,1,fp8,fp8,0,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,128,0,1,float16,fp8,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,128,0,1,float16,fp8,0,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,128,0,1,fp8,fp8,0,0.015813333292802174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,128,0,1,float16,fp8,0,0.014357333381970724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,128,0,1,float16,float16,0,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,128,0,1,float16,fp8,0,0.01393066719174385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,128,0,1,fp8,fp8,0,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,128,0,1,float16,float16,0,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,128,0,1,fp8,fp8,0,0.016330666840076447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,128,0,1,float16,float16,0,0.014122666170199713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,128,0,1,fp8,fp8,0,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,128,0,1,fp8,fp8,0,10.888404846191406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,128,0,1,fp8,fp8,0,10.974005381266275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,128,0,1,float16,float16,0,16.392874399820965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,128,0,1,float16,fp8,0,16.22523244222005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,128,0,1,float16,float16,0,16.26404317220052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,128,0,1,float16,fp8,0,16.533541361490887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,128,0,1,float16,float16,0,16.45033009847005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,128,0,1,float16,fp8,0,16.51318359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,128,0,1,fp8,fp8,0,11.019583384195963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,128,0,1,fp8,fp8,0,5.771312077840169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,128,0,1,float16,fp8,0,8.466181437174479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,128,0,1,float16,float16,0,8.495306650797525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,128,0,1,fp8,fp8,0,11.103903452555338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,128,0,1,float16,float16,0,8.278095881144205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,128,0,1,fp8,fp8,0,5.629109064737956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,128,0,1,float16,fp8,0,9.174847920735678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,128,0,1,float16,float16,0,18.220298767089844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,128,0,1,float16,fp8,0,18.72757339477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,128,0,1,fp8,fp8,0,6.445440292358398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,128,0,1,float16,float16,0,9.466133117675781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,128,0,1,float16,fp8,0,9.582490921020508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,128,0,1,float16,float16,0,8.435920079549154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,128,0,1,fp8,fp8,0,5.744885126749675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,128,0,1,float16,fp8,0,10.003920237223307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,128,0,1,float16,fp8,0,4.604602813720703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,128,0,1,float16,float16,0,4.526965459187825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,128,0,1,fp8,fp8,0,3.0836639404296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,128,0,1,fp8,fp8,0,5.891541163126628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,128,0,1,float16,float16,0,8.607738494873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,128,0,1,float16,fp8,0,8.718026479085287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,128,0,1,float16,float16,0,4.283983866373698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,128,0,1,fp8,fp8,0,2.831637382507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,128,0,1,fp8,fp8,0,2.8295199076334634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,128,0,1,float16,fp8,0,4.298783938090007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,128,0,1,float16,float16,0,4.1769758860270185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,128,0,1,float16,fp8,0,4.406309445699056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,128,0,1,float16,float16,0,4.516501426696777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,128,0,1,fp8,fp8,0,2.8229331970214844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,128,0,1,float16,fp8,0,4.163162549336751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,128,0,1,float16,float16,0,2.273402690887451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,128,0,1,float16,float16,0,4.431610743204753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,128,0,1,fp8,fp8,0,3.047856012980143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,128,0,1,fp8,fp8,0,1.5219732920328777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,128,0,1,float16,fp8,0,2.223893324534098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,128,0,1,float16,float16,0,2.1571680704752603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,128,0,1,float16,fp8,0,4.265066782633464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,128,0,1,float16,fp8,0,2.1539573669433594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,128,0,1,fp8,fp8,0,1.433674653371175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,128,0,1,fp8,fp8,0,1.4357813199361165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,128,0,1,float16,float16,0,2.1428213119506836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,128,0,1,float16,fp8,0,2.1562506357828775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,128,0,1,fp8,fp8,0,1.4420852661132812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,128,0,1,float16,float16,0,2.2150986989339194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,128,0,1,float16,fp8,0,2.209503968556722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,128,0,1,float16,float16,0,2.1137332916259766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,128,0,1,fp8,fp8,0,1.4726719856262207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,128,0,1,float16,fp8,0,2.1728320121765137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,128,0,1,fp8,fp8,0,6.447071711222331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,128,0,1,fp8,fp8,0,6.652005513509114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,128,0,1,float16,float16,0,9.750693639119467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,128,0,1,float16,fp8,0,9.580570856730143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,128,0,1,float16,fp8,0,9.589029312133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,128,0,1,float16,float16,0,9.962991714477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,128,0,1,float16,float16,0,9.546501159667969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,128,0,1,float16,fp8,0,9.569269180297852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,128,0,1,fp8,fp8,0,6.782463709513347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,128,0,1,fp8,fp8,0,3.3986454010009766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,128,0,1,float16,float16,0,5.036933263142903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,128,0,1,float16,fp8,0,5.489205042521159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,128,0,1,fp8,fp8,0,6.603904088338216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,128,0,1,float16,float16,0,4.868490537007649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,128,0,1,float16,float16,0,9.48523203531901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,128,0,1,fp8,fp8,0,3.2915948232014975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,128,0,1,float16,fp8,0,5.082234700520833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,128,0,1,float16,fp8,0,10.348527908325195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,128,0,1,fp8,fp8,0,3.5407946904500327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,128,0,1,float16,float16,0,4.92087999979655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,128,0,1,float16,fp8,0,4.925941467285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,128,0,1,float16,float16,0,5.132517178853353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,128,0,1,fp8,fp8,0,3.4394025802612305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,128,0,1,float16,fp8,0,5.143093427022298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,128,0,1,fp8,fp8,0,3.6230878829956055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,128,0,1,float16,float16,0,5.185514767964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,128,0,1,float16,float16,0,2.5666133562723794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,128,0,1,fp8,fp8,0,1.745589256286621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,128,0,1,float16,fp8,0,2.5872960090637207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,128,0,1,float16,fp8,0,5.291418711344401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,128,0,1,float16,float16,0,2.543839931488037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,128,0,1,fp8,fp8,0,1.6899147033691406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,128,0,1,fp8,fp8,0,1.6909386316935222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,128,0,1,float16,fp8,0,2.4343093236287436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,128,0,1,float16,float16,0,2.5007146199544272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,128,0,1,float16,fp8,0,2.5530079205830893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,128,0,1,fp8,fp8,0,1.6381653149922688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,128,0,1,float16,float16,0,2.508351961771647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,128,0,1,float16,fp8,0,2.4405973752339682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,128,0,1,fp8,fp8,0,0.8877759774525961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,128,0,1,fp8,fp8,0,1.6645867029825847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,128,0,1,float16,float16,0,1.3345759709676106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,128,0,1,float16,fp8,0,1.3245920340220134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,128,0,1,float16,float16,0,2.501584053039551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,128,0,1,float16,float16,0,1.3264053662618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,128,0,1,float16,fp8,0,2.6251413027445474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,128,0,1,fp8,fp8,0,0.8291680018107096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,128,0,1,float16,fp8,0,1.3540159861246746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,128,0,1,fp8,fp8,0,0.9125706354777018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,128,0,1,float16,float16,0,1.2606186866760254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,128,0,1,float16,fp8,0,1.2459572950998943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,128,0,1,float16,float16,0,1.2960480054219563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,128,0,1,fp8,fp8,0,0.8151199817657471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,128,0,1,float16,fp8,0,1.3230506579081218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,128,0,1,float16,float16,0,1.2519520123799641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,128,0,1,fp8,fp8,0,0.851919968922933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,128,0,1,float16,fp8,0,1.25163729985555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,128,0,1,fp8,fp8,0,4.837413469950358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,128,0,1,fp8,fp8,0,4.747413317362468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,128,0,1,float16,float16,0,6.754271825154622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,128,0,1,float16,fp8,0,7.216298421223958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,128,0,1,float16,float16,0,6.824319839477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,128,0,1,float16,fp8,0,6.8122507731119795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,128,0,1,float16,float16,0,6.914373397827148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,128,0,1,fp8,fp8,0,5.070010821024577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,128,0,1,fp8,fp8,0,2.4597813288370767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,128,0,1,float16,float16,0,3.55947208404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,128,0,1,float16,fp8,0,3.6053921381632485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,128,0,1,float16,fp8,0,6.914682388305664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,128,0,1,fp8,fp8,0,4.696799914042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,128,0,1,float16,float16,0,6.915381113688151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,128,0,1,float16,float16,0,3.5523465474446616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,128,0,1,fp8,fp8,0,2.301781336466471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,128,0,1,float16,fp8,0,7.308154424031575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,128,0,1,float16,fp8,0,3.4690933227539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,128,0,1,fp8,fp8,0,2.2443200747172036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,128,0,1,float16,float16,0,3.3238614400227866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,128,0,1,float16,fp8,0,3.4575573603312173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,128,0,1,fp8,fp8,0,2.342458724975586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,128,0,1,float16,float16,0,3.527359962463379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,128,0,1,float16,float16,0,1.7791892687479656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,128,0,1,float16,fp8,0,1.7289759318033855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,128,0,1,fp8,fp8,0,2.3747520446777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,128,0,1,float16,fp8,0,3.486037254333496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,128,0,1,float16,float16,0,3.6193065643310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,128,0,1,float16,fp8,0,3.4946667353312173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,128,0,1,fp8,fp8,0,1.2886559963226318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,128,0,1,float16,float16,0,1.716250737508138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,128,0,1,fp8,fp8,0,1.1704213619232178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,128,0,1,float16,fp8,0,1.6959519386291504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,128,0,1,fp8,fp8,0,1.3145439624786377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,128,0,1,float16,float16,0,1.6803679466247559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,128,0,1,float16,fp8,0,1.6702666282653809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,128,0,1,fp8,fp8,0,1.1689813137054443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,128,0,1,float16,float16,0,1.7114399274190266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,128,0,1,float16,fp8,0,1.6812426249186199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,128,0,1,float16,float16,0,1.7298879623413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,128,0,1,float16,fp8,0,0.9228853384653727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,128,0,1,float16,float16,0,0.9155893325805664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,128,0,1,fp8,fp8,0,0.6208639939626058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,128,0,1,fp8,fp8,0,1.2019946575164795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,128,0,1,float16,float16,0,0.9027946790059408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,128,0,1,float16,fp8,0,1.7006932894388835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,128,0,1,float16,fp8,0,0.9247999986012777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,128,0,1,fp8,fp8,0,0.6284639835357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,128,0,1,fp8,fp8,0,0.5960906744003296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,128,0,1,float16,float16,0,0.9004693031311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,128,0,1,float16,fp8,0,0.9000960191090902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,128,0,1,float16,float16,0,0.9184532960255941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,128,0,1,fp8,fp8,0,0.6155466636021932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,128,0,1,float16,fp8,0,0.9032213687896729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,128,0,1,float16,float16,0,0.914410670598348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,128,0,1,fp8,fp8,0,0.6172000169754028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,128,0,1,float16,fp8,0,0.9300426642100016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,128,0,1,fp8,fp8,0,6.093514760335286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,128,0,1,fp8,fp8,0,6.171301523844401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,128,0,1,float16,float16,0,8.903807957967123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,128,0,1,float16,fp8,0,8.801034927368164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,128,0,1,float16,fp8,0,8.988853454589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,128,0,1,float16,float16,0,9.425008138020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,128,0,1,float16,float16,0,9.189557393391928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,128,0,1,float16,fp8,0,9.283231735229492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,128,0,1,fp8,fp8,0,3.278437296549479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,128,0,1,fp8,fp8,0,6.253023783365886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,128,0,1,float16,float16,0,4.6632639567057295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,128,0,1,float16,fp8,0,4.9389387766520185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,128,0,1,float16,float16,0,4.723498662312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,128,0,1,fp8,fp8,0,6.603434880574544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,128,0,1,float16,float16,0,9.198954900105795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,128,0,1,fp8,fp8,0,3.1069065729777017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,128,0,1,float16,fp8,0,4.561546643575032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,128,0,1,float16,fp8,0,9.23907725016276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,128,0,1,fp8,fp8,0,3.1619625091552734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,128,0,1,float16,float16,0,5.063018798828125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,128,0,1,float16,fp8,0,4.497962633768718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,128,0,1,fp8,fp8,0,3.2273918787638345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,128,0,1,float16,float16,0,4.669365247090657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,128,0,1,float16,fp8,0,4.611904144287109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,128,0,1,fp8,fp8,0,3.272202809651693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,128,0,1,float16,fp8,0,2.369605382283529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,128,0,1,fp8,fp8,0,1.6057653427124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,128,0,1,float16,float16,0,2.5217226346333823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,128,0,1,float16,float16,0,4.712522824605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,128,0,1,float16,fp8,0,4.829071998596191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,128,0,1,float16,float16,0,2.3044212659200034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,128,0,1,float16,fp8,0,2.213066736857096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,128,0,1,fp8,fp8,0,1.6230613390604656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,128,0,1,fp8,fp8,0,1.4965119361877441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,128,0,1,float16,float16,0,2.286224047342936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,128,0,1,float16,fp8,0,2.3108533223470054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,128,0,1,float16,float16,0,2.2596799532572427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,128,0,1,fp8,fp8,0,1.5509066581726074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,128,0,1,float16,fp8,0,2.2636586825052896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,128,0,1,float16,float16,0,2.2695466677347818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,128,0,1,float16,float16,0,1.20469864209493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,128,0,1,fp8,fp8,0,1.621338685353597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,128,0,1,float16,fp8,0,2.2692853609720864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,128,0,1,fp8,fp8,0,0.8440159956614176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,128,0,1,float16,fp8,0,1.2302239735921223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,128,0,1,float16,float16,0,1.1555253664652507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,128,0,1,fp8,fp8,0,0.8265493710835775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,128,0,1,float16,fp8,0,1.1508853435516357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,128,0,1,fp8,fp8,0,0.7979199886322021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,128,0,1,float16,float16,0,1.1395786603291829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,128,0,1,float16,fp8,0,1.1556479930877686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,128,0,1,float16,float16,0,1.140384038289388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,128,0,1,float16,fp8,0,1.13209064801534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,128,0,1,fp8,fp8,0,0.8159306844075521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,128,0,1,float16,float16,0,1.1770719687143962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,128,0,1,fp8,fp8,0,0.8131840229034424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,128,0,1,float16,fp8,0,1.1885120073954265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,128,0,1,fp8,fp8,0,0.4485866626103719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,128,0,1,float16,float16,0,0.6454293330510458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,128,0,1,float16,fp8,0,0.6607626676559448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,128,0,1,float16,float16,0,0.6189119815826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,128,0,1,fp8,fp8,0,0.40753066539764404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,128,0,1,float16,fp8,0,0.6371093193689982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,128,0,1,float16,float16,0,0.6331626574198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,128,0,1,fp8,fp8,0,0.41973865032196045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,128,0,1,float16,fp8,0,0.6331093311309814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,128,0,1,float16,float16,0,0.6192053159077963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,128,0,1,fp8,fp8,0,0.410373330116272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,128,0,1,float16,fp8,0,0.6308960119883219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,128,0,1,float16,float16,0,0.6341013511021932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,128,0,1,fp8,fp8,0,0.42475732167561847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,128,0,1,float16,fp8,0,0.624730666478475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,128,0,1,fp8,fp8,0,3.8530454635620117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,128,0,1,fp8,fp8,0,3.768789291381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,128,0,1,float16,float16,0,5.293957392374675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,128,0,1,float16,fp8,0,5.359647750854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,128,0,1,float16,fp8,0,5.413866678873698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,128,0,1,float16,float16,0,5.913978576660156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,128,0,1,float16,float16,0,5.487909317016602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,128,0,1,fp8,fp8,0,3.6627626419067383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,128,0,1,float16,fp8,0,5.3748321533203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,128,0,1,fp8,fp8,0,1.8788426717122395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,128,0,1,float16,float16,0,2.6843786239624023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,128,0,1,float16,fp8,0,2.8845866521199546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,128,0,1,float16,float16,0,5.4134775797526045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,128,0,1,fp8,fp8,0,3.7418079376220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,128,0,1,float16,float16,0,2.7831039428710938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,128,0,1,fp8,fp8,0,1.8395147323608398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,128,0,1,float16,fp8,0,2.5548532803853354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,128,0,1,float16,fp8,0,5.619818369547526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,128,0,1,fp8,fp8,0,1.7803680102030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,128,0,1,float16,float16,0,2.6459199587504068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,128,0,1,float16,fp8,0,2.721343994140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,128,0,1,float16,float16,0,2.6837441126505532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,128,0,1,fp8,fp8,0,1.850661277770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,128,0,1,float16,fp8,0,2.7031520207722983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,128,0,1,float16,float16,0,1.3755359649658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,128,0,1,float16,fp8,0,1.4285227457682292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,128,0,1,fp8,fp8,0,1.846186637878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,128,0,1,fp8,fp8,0,1.0742080211639404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,128,0,1,float16,float16,0,2.7024428049723306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,128,0,1,float16,fp8,0,2.6338507334391275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,128,0,1,float16,float16,0,1.2987093130747478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,128,0,1,fp8,fp8,0,0.9529493649800619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,128,0,1,float16,fp8,0,1.302677313486735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,128,0,1,fp8,fp8,0,0.9143946965535482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,128,0,1,float16,float16,0,1.370805263519287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,128,0,1,float16,fp8,0,1.3149173259735107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,128,0,1,float16,float16,0,1.3161280155181885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,128,0,1,fp8,fp8,0,0.9670666853586832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,128,0,1,float16,fp8,0,1.3535466194152832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,128,0,1,float16,float16,0,0.7260639667510986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,128,0,1,fp8,fp8,0,0.93832000096639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,128,0,1,float16,fp8,0,0.7113920052846273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,128,0,1,fp8,fp8,0,0.5134720007578532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,128,0,1,float16,float16,0,1.3196319739023845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,128,0,1,float16,fp8,0,1.3512427012125652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,128,0,1,float16,float16,0,0.6843252976735433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,128,0,1,fp8,fp8,0,0.5006933212280273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,128,0,1,float16,fp8,0,0.691152016321818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,128,0,1,float16,float16,0,0.6754186948140463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,128,0,1,float16,fp8,0,0.6820159753163656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,128,0,1,fp8,fp8,0,0.47041598955790204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,128,0,1,float16,float16,0,0.6917440096537272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,128,0,1,fp8,fp8,0,0.4824639956156413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,128,0,1,float16,fp8,0,0.6981813112894694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,128,0,1,float16,float16,0,0.7051040331522623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,128,0,1,float16,fp8,0,0.35634132226308185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,128,0,1,float16,float16,0,0.34809601306915283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,128,0,1,fp8,fp8,0,0.4875946839650472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,128,0,1,float16,fp8,0,0.701632022857666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,128,0,1,fp8,fp8,0,0.2812053362528483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,128,0,1,float16,float16,0,0.3386186758677165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,128,0,1,float16,fp8,0,0.3399893442789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,128,0,1,fp8,fp8,0,0.2651253342628479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,128,0,1,float16,float16,0,0.3388426701227824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,128,0,1,float16,fp8,0,0.33507732550303143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,128,0,1,fp8,fp8,0,0.2667626738548279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,128,0,1,float16,float16,0,0.3425439993540446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,128,0,1,float16,fp8,0,0.34401599566141766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,128,0,1,fp8,fp8,0,0.26607465744018555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,128,0,1,float16,float16,0,0.34770135084788006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,128,0,1,float16,fp8,0,0.34588801860809326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,128,0,1,fp8,fp8,0,0.2701066732406616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,128,0,1,fp8,fp8,0,3.5739787419637046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,128,0,1,fp8,fp8,0,3.661386807759603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,128,0,1,float16,float16,0,5.1495466232299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,128,0,1,float16,fp8,0,5.172922770182292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,128,0,1,float16,float16,0,5.20684814453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,128,0,1,float16,fp8,0,5.284133275349935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,128,0,1,float16,float16,0,5.324240048726399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,128,0,1,fp8,fp8,0,3.6995038986206055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,128,0,1,float16,float16,0,2.669935862223307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,128,0,1,float16,fp8,0,2.694682757059733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,128,0,1,float16,fp8,0,5.295685450236003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,128,0,1,fp8,fp8,0,3.8335094451904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,128,0,1,fp8,fp8,0,1.9430774052937825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,128,0,1,float16,float16,0,5.195568084716797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,128,0,1,float16,fp8,0,5.329920132954915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,128,0,1,fp8,fp8,0,1.8344586690266926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,128,0,1,float16,float16,0,2.433285395304362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,128,0,1,float16,fp8,0,2.5803839365641275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,128,0,1,fp8,fp8,0,1.765493392944336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,128,0,1,float16,float16,0,2.459808031717936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,128,0,1,float16,fp8,0,2.5338026682535806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,128,0,1,fp8,fp8,0,1.8333652814229329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,128,0,1,float16,float16,0,2.5399999618530273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,128,0,1,float16,float16,0,1.3155732949574788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,128,0,1,float16,fp8,0,2.5271679560343423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,128,0,1,fp8,fp8,0,1.8558026949564617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,128,0,1,float16,fp8,0,1.3188532988230388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,128,0,1,float16,float16,0,2.5836373964945474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,128,0,1,float16,fp8,0,2.69759464263916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,128,0,1,fp8,fp8,0,0.981386661529541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,128,0,1,fp8,fp8,0,0.9244639873504639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,128,0,1,float16,float16,0,1.195520003636678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,128,0,1,float16,fp8,0,1.2393546899159749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,128,0,1,float16,float16,0,1.2199947039286296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,128,0,1,fp8,fp8,0,0.8967680136362711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,128,0,1,float16,fp8,0,1.2108159859975178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,128,0,1,fp8,fp8,0,0.9303733507792155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,128,0,1,float16,float16,0,1.2479573090871174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,128,0,1,float16,fp8,0,1.224186658859253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,128,0,1,fp8,fp8,0,0.952613353729248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,128,0,1,float16,float16,0,1.2911946773529053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,128,0,1,float16,fp8,0,1.256053368250529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,128,0,1,float16,float16,0,0.681663990020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,128,0,1,fp8,fp8,0,0.5245333512624105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,128,0,1,float16,fp8,0,0.6891733010609945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,128,0,1,float16,float16,0,0.6435626745223999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,128,0,1,float16,fp8,0,0.6450826724370321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,128,0,1,fp8,fp8,0,0.4836213191350301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,128,0,1,float16,float16,0,0.6335093180338541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,128,0,1,fp8,fp8,0,0.47590935230255127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,128,0,1,float16,fp8,0,0.6412320137023926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,128,0,1,float16,float16,0,0.6424160003662109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,128,0,1,float16,fp8,0,0.6466186841328939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,128,0,1,fp8,fp8,0,0.489408016204834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,128,0,1,float16,float16,0,0.6553066571553549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,128,0,1,float16,fp8,0,0.6554239988327026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,128,0,1,float16,float16,0,0.36737600962320965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,128,0,1,fp8,fp8,0,0.49057598908742267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,128,0,1,float16,fp8,0,0.37401068210601807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,128,0,1,fp8,fp8,0,0.2731093366940816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,128,0,1,float16,float16,0,0.35651199022928876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,128,0,1,float16,float16,0,0.3547786474227905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,128,0,1,fp8,fp8,0,0.2497226595878601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,128,0,1,float16,fp8,0,0.35683198769887287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,128,0,1,float16,fp8,0,0.3596213261286418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,128,0,1,fp8,fp8,0,0.25591466824213666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,128,0,1,float16,fp8,0,0.3558719952901204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,128,0,1,fp8,fp8,0,0.24595733483632407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,128,0,1,float16,float16,0,0.3588053385416667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,128,0,1,float16,float16,0,0.3601226806640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,128,0,1,float16,fp8,0,0.3654293219248454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,128,0,1,fp8,fp8,0,0.2608213424682617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,128,0,1,float16,fp8,0,0.18013866742451987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,128,0,1,float16,float16,0,0.18609599272410074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,128,0,1,float16,fp8,0,0.18897066513697305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,128,0,1,fp8,fp8,0,0.15461867054303488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,128,0,1,float16,float16,0,0.179365336894989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,128,0,1,fp8,fp8,0,0.14435733358065286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,128,0,1,float16,float16,0,0.17945067087809244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,128,0,1,float16,fp8,0,0.1809013287226359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,128,0,1,fp8,fp8,0,0.14577600359916687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,128,0,1,float16,float16,0,0.18081599473953247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,128,0,1,float16,fp8,0,0.18144534031550089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,128,0,1,fp8,fp8,0,0.14780799547831217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,128,0,1,float16,float16,0,0.18288000424702963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,128,0,1,float16,fp8,0,0.18346667289733887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,128,0,1,fp8,fp8,0,0.15061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,128,0,1,fp8,fp8,0,2.234293301900228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,128,0,1,fp8,fp8,0,2.2595787048339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,128,0,1,float16,float16,0,3.1251147588094077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,128,0,1,float16,fp8,0,3.0915307998657227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,128,0,1,float16,float16,0,3.106570561726888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,128,0,1,float16,fp8,0,3.1133225758870444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,128,0,1,float16,float16,0,3.1155627568562827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,128,0,1,float16,float16,0,1.6311465899149578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,128,0,1,fp8,fp8,0,2.274496078491211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,128,0,1,float16,fp8,0,3.1686347325642905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,128,0,1,float16,fp8,0,1.7174879709879558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,128,0,1,fp8,fp8,0,2.3704479535420737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,128,0,1,float16,float16,0,3.3005758921305337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,128,0,1,float16,fp8,0,3.226352055867513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,128,0,1,fp8,fp8,0,1.247157335281372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,128,0,1,float16,float16,0,1.4573493003845215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,128,0,1,fp8,fp8,0,1.1502186457316081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,128,0,1,float16,fp8,0,1.4778453509012859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,128,0,1,float16,float16,0,1.4639093081156414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,128,0,1,fp8,fp8,0,1.1512373288472493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,128,0,1,float16,fp8,0,1.5020373662312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,128,0,1,float16,float16,0,1.5169973373413086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,128,0,1,fp8,fp8,0,1.1767040093739827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,128,0,1,float16,fp8,0,1.529861291249593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,128,0,1,float16,float16,0,0.8450026512145996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,128,0,1,fp8,fp8,0,1.1709333260854085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,128,0,1,float16,float16,0,1.530197302500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,128,0,1,fp8,fp8,0,0.6449546813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,128,0,1,float16,fp8,0,1.5713866551717122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,128,0,1,float16,fp8,0,0.8443199793497721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,128,0,1,float16,float16,0,0.768234650293986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,128,0,1,fp8,fp8,0,0.5868159929911295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,128,0,1,float16,fp8,0,0.7437280019124349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,128,0,1,fp8,fp8,0,0.5797066688537598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,128,0,1,float16,float16,0,0.7413866519927979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,128,0,1,float16,fp8,0,0.7455573081970215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,128,0,1,float16,float16,0,0.7709066867828369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,128,0,1,float16,fp8,0,0.7649493217468262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,128,0,1,fp8,fp8,0,0.5882346630096436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,128,0,1,float16,float16,0,0.7880106767018636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,128,0,1,float16,float16,0,0.4333653450012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,128,0,1,float16,fp8,0,0.44150932629903156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,128,0,1,fp8,fp8,0,0.34115199247996014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,128,0,1,fp8,fp8,0,0.6051466862360636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,128,0,1,float16,fp8,0,0.7758986949920654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,128,0,1,float16,float16,0,0.40008000532786053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,128,0,1,float16,fp8,0,0.4013813336690267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,128,0,1,fp8,fp8,0,0.29792000850041706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,128,0,1,float16,float16,0,0.4048373301823934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,128,0,1,float16,fp8,0,0.40434666474660236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,128,0,1,fp8,fp8,0,0.3021226723988851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,128,0,1,float16,fp8,0,0.40662399927775067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,128,0,1,float16,float16,0,0.40625067551930744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,128,0,1,fp8,fp8,0,0.30182933807373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,128,0,1,float16,fp8,0,0.22228266795476279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,128,0,1,float16,float16,0,0.41302935282389325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,128,0,1,float16,fp8,0,0.41995731989542645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,128,0,1,float16,float16,0,0.22150399287541708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,128,0,1,fp8,fp8,0,0.16522666811943054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,128,0,1,fp8,fp8,0,0.3169386585553487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,128,0,1,fp8,fp8,0,0.18261865774790445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,128,0,1,float16,float16,0,0.20203733444213867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,128,0,1,float16,fp8,0,0.20274666945139566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,128,0,1,float16,float16,0,0.20254933834075928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,128,0,1,float16,fp8,0,0.2059146761894226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,128,0,1,fp8,fp8,0,0.16569599509239197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,128,0,1,float16,float16,0,0.2058453361193339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,128,0,1,float16,fp8,0,0.20854399601618448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,128,0,1,fp8,fp8,0,0.16839466492335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,128,0,1,float16,float16,0,0.20972265799840292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,128,0,1,float16,fp8,0,0.21141332387924194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,128,0,1,fp8,fp8,0,0.17303466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,128,0,1,float16,float16,0,0.1280586620171865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,128,0,1,float16,float16,0,0.12136000394821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,128,0,1,float16,fp8,0,0.12890666723251343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,128,0,1,fp8,fp8,0,0.10822400450706482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,128,0,1,float16,float16,0,0.12205866972605388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,128,0,1,float16,float16,0,0.12247467041015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,128,0,1,float16,fp8,0,0.12229866782824199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,128,0,1,fp8,fp8,0,0.09757866462071736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,128,0,1,float16,fp8,0,0.121888001759847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,128,0,1,fp8,fp8,0,0.0990559955437978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,128,0,1,float16,fp8,0,0.1228053371111552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,128,0,1,fp8,fp8,0,0.10114133358001709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,128,0,1,float16,float16,0,0.12444266676902771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,128,0,1,float16,fp8,0,0.1260693371295929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,128,0,1,fp8,fp8,0,0.10372799634933472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,128,0,1,fp8,fp8,0,2.4131253560384116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,128,0,1,float16,float16,0,3.0878559748331704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,128,0,1,float16,fp8,0,3.171669324239095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,128,0,1,fp8,fp8,0,2.420421282450358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,128,0,1,float16,float16,0,3.1692212422688804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,128,0,1,float16,fp8,0,3.11356258392334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,128,0,1,float16,float16,0,3.2296320597330728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,128,0,1,float16,fp8,0,3.175413449605306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,128,0,1,fp8,fp8,0,2.461695988972982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,128,0,1,float16,float16,0,3.3636000951131186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,128,0,1,float16,float16,0,1.7257812817891438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,128,0,1,fp8,fp8,0,1.390394687652588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,128,0,1,float16,fp8,0,1.703104019165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,128,0,1,float16,fp8,0,3.250080108642578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,128,0,1,fp8,fp8,0,2.5328799883524575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,128,0,1,float16,float16,0,1.528783957163493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,128,0,1,float16,fp8,0,1.5411146481831868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,128,0,1,fp8,fp8,0,1.2332266966501872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,128,0,1,float16,float16,0,1.558303991953532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,128,0,1,fp8,fp8,0,1.242240031560262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,128,0,1,float16,fp8,0,1.5415679613749187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,128,0,1,fp8,fp8,0,1.2513226668039958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,128,0,1,float16,float16,0,1.5573973655700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,128,0,1,float16,fp8,0,1.5495947202046711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,128,0,1,float16,float16,0,1.6453973452250164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,128,0,1,float16,float16,0,0.8721493085225424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,128,0,1,float16,fp8,0,1.6215999921162922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,128,0,1,fp8,fp8,0,1.3016266822814941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,128,0,1,fp8,fp8,0,0.7038453420003256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,128,0,1,float16,fp8,0,0.8730133374532064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,128,0,1,float16,float16,0,0.7513333161671957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,128,0,1,fp8,fp8,0,0.613098661104838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,128,0,1,float16,fp8,0,0.7647733688354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,128,0,1,float16,float16,0,0.7706133524576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,128,0,1,float16,float16,0,0.7803306579589844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,128,0,1,float16,fp8,0,0.7735466957092285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,128,0,1,float16,fp8,0,0.7613066832224528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,128,0,1,fp8,fp8,0,0.6312959988911947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,128,0,1,fp8,fp8,0,0.6174666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,128,0,1,float16,float16,0,0.8041760126749674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,128,0,1,float16,float16,0,0.45051733652750653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,128,0,1,fp8,fp8,0,0.6645333369572958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,128,0,1,float16,fp8,0,0.4493173360824585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,128,0,1,fp8,fp8,0,0.369920015335083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,128,0,1,float16,fp8,0,0.803002675374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,128,0,1,float16,float16,0,0.3929920196533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,128,0,1,float16,fp8,0,0.3972373406092326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,128,0,1,fp8,fp8,0,0.3259359995524089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,128,0,1,float16,float16,0,0.4027413527170817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,128,0,1,float16,fp8,0,0.40057599544525146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,128,0,1,fp8,fp8,0,0.32870399951934814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,128,0,1,float16,float16,0,0.41074665387471515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,128,0,1,float16,fp8,0,0.4045013189315796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,128,0,1,fp8,fp8,0,0.3330293297767639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,128,0,1,fp8,fp8,0,0.346127986907959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,128,0,1,float16,float16,0,0.415333350499471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,128,0,1,float16,fp8,0,0.4210346539815267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,128,0,1,float16,float16,0,0.244869331518809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,128,0,1,float16,fp8,0,0.24651199579238892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,128,0,1,fp8,fp8,0,0.19533334175745645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,128,0,1,float16,float16,0,0.21901865800221762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,128,0,1,float16,fp8,0,0.22218666474024454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,128,0,1,fp8,fp8,0,0.1686506668726603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,128,0,1,float16,float16,0,0.22314133246739706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,128,0,1,float16,fp8,0,0.22307199239730835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,128,0,1,fp8,fp8,0,0.1718399922053019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,128,0,1,float16,float16,0,0.22489066918691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,128,0,1,float16,fp8,0,0.22542399168014526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,128,0,1,fp8,fp8,0,0.17373865842819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,128,0,1,float16,float16,0,0.23100266853968301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,128,0,1,float16,fp8,0,0.23067732652028403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,128,0,1,fp8,fp8,0,0.1819360057512919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,128,0,1,float16,float16,0,0.1246453324953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,128,0,1,float16,fp8,0,0.12615467111269632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,128,0,1,fp8,fp8,0,0.10916800300280254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,128,0,1,float16,float16,0,0.1122826635837555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,128,0,1,float16,fp8,0,0.1132319966952006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,128,0,1,fp8,fp8,0,0.09701333443323772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,128,0,1,float16,float16,0,0.11307733257611592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,128,0,1,float16,fp8,0,0.11462400356928508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,128,0,1,fp8,fp8,0,0.09782399733861287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,128,0,1,float16,float16,0,0.11526933312416077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,128,0,1,float16,fp8,0,0.11516799529393514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,128,0,1,fp8,fp8,0,0.09880533814430237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,128,0,1,float16,float16,0,0.11674132943153381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,128,0,1,float16,fp8,0,0.11958400408426921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,128,0,1,fp8,fp8,0,0.10254399975140889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,128,0,1,float16,float16,0,0.07294933497905731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,128,0,1,float16,fp8,0,0.07378666599591573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,128,0,1,fp8,fp8,0,0.06710400183995564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,128,0,1,float16,float16,0,0.06905599931875865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,128,0,1,float16,fp8,0,0.06935466825962067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,128,0,1,float16,fp8,0,0.06949866811434428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,128,0,1,fp8,fp8,0,0.060138667623202004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,128,0,1,fp8,fp8,0,0.05949333310127258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,128,0,1,float16,float16,0,0.06938133140405019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,128,0,1,float16,fp8,0,0.06925333539644878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,128,0,1,fp8,fp8,0,0.05987200140953064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,128,0,1,float16,float16,0,0.06904000043869019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,128,0,1,float16,float16,0,0.07049599786599477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,128,0,1,float16,fp8,0,0.07083733379840851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,128,0,1,fp8,fp8,0,0.06233066817124685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,128,0,1,fp8,fp8,0,1.5946186383565266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,128,0,1,float16,float16,0,1.9407040278116863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,128,0,1,float16,fp8,0,1.9173226356506348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,128,0,1,fp8,fp8,0,1.6231892903645833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,128,0,1,float16,float16,0,1.9631840387980144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,128,0,1,float16,fp8,0,1.9363412857055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,128,0,1,float16,float16,0,1.9864586194356282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,128,0,1,float16,fp8,0,1.9975840250651042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,128,0,1,fp8,fp8,0,1.626021385192871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,128,0,1,float16,float16,0,1.1362026532491047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,128,0,1,float16,fp8,0,1.1158506870269775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,128,0,1,fp8,fp8,0,0.9519893328348795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,128,0,1,float16,float16,0,2.0761119524637857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,128,0,1,fp8,fp8,0,1.7042826016743977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,128,0,1,float16,float16,0,0.972325325012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,128,0,1,float16,fp8,0,2.053760051727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,128,0,1,float16,fp8,0,0.960197369257609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,128,0,1,fp8,fp8,0,0.8120906352996826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,128,0,1,float16,float16,0,0.9776480197906494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,128,0,1,float16,fp8,0,0.9892000357309977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,128,0,1,fp8,fp8,0,0.8070399761199951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,128,0,1,float16,float16,0,0.993717352549235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,128,0,1,float16,fp8,0,0.9921600023905436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,128,0,1,fp8,fp8,0,0.8274933497111002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,128,0,1,float16,float16,0,0.5705493291219076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,128,0,1,float16,float16,0,1.0333813031514485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,128,0,1,float16,fp8,0,1.0216320355733235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,128,0,1,fp8,fp8,0,0.8638559977213541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,128,0,1,fp8,fp8,0,0.4914506673812866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,128,0,1,float16,fp8,0,0.5699413220087687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,128,0,1,float16,float16,0,0.4846186637878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,128,0,1,float16,fp8,0,0.4901653528213501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,128,0,1,fp8,fp8,0,0.41894932587941486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,128,0,1,float16,float16,0,0.4915359814961751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,128,0,1,float16,fp8,0,0.49478399753570557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,128,0,1,fp8,fp8,0,0.4278613328933716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,128,0,1,float16,float16,0,0.49882666269938153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,128,0,1,float16,fp8,0,0.49826665719350177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,128,0,1,fp8,fp8,0,0.42974400520324707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,128,0,1,float16,float16,0,0.5196160078048706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,128,0,1,float16,fp8,0,0.5193973382314047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,128,0,1,fp8,fp8,0,0.4496266841888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,128,0,1,fp8,fp8,0,0.26632533470789593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,128,0,1,float16,float16,0,0.30407466491063434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,128,0,1,float16,fp8,0,0.30344533920288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,128,0,1,float16,float16,0,0.2621120015780131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,128,0,1,float16,fp8,0,0.26395734151204425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,128,0,1,fp8,fp8,0,0.2112906575202942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,128,0,1,float16,float16,0,0.2656000057856242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,128,0,1,float16,fp8,0,0.2638559937477112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,128,0,1,fp8,fp8,0,0.21460266908009848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,128,0,1,fp8,fp8,0,0.2190879980723063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,128,0,1,float16,float16,0,0.26837867498397827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,128,0,1,float16,fp8,0,0.2690986593564351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,128,0,1,float16,float16,0,0.2772746682167053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,128,0,1,float16,fp8,0,0.28008532524108887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,128,0,1,fp8,fp8,0,0.2270453373591105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,128,0,1,float16,float16,0,0.1638879974683126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,128,0,1,float16,fp8,0,0.16387733817100525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,128,0,1,fp8,fp8,0,0.13743999600410461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,128,0,1,float16,float16,0,0.13647466897964478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,128,0,1,float16,fp8,0,0.13754666845003763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,128,0,1,fp8,fp8,0,0.11964799960454305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,128,0,1,float16,float16,0,0.13661332925160727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,128,0,1,fp8,fp8,0,0.12031466762224834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,128,0,1,float16,fp8,0,0.1395680010318756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,128,0,1,float16,float16,0,0.1388159990310669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,128,0,1,float16,fp8,0,0.14112533132235208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,128,0,1,fp8,fp8,0,0.12076800068219502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,128,0,1,float16,float16,0,0.14478933811187744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,128,0,1,float16,fp8,0,0.14586666226387024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,128,0,1,fp8,fp8,0,0.12626666824022928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,128,0,1,fp8,fp8,0,0.06863466898600261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,128,0,1,float16,float16,0,0.08685866991678874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,128,0,1,float16,fp8,0,0.08958933750788371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,128,0,1,fp8,fp8,0,0.07905066510041554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,128,0,1,float16,float16,0,0.07751466830571492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,128,0,1,float16,fp8,0,0.07762133578459422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,128,0,1,float16,float16,0,0.0774293343226115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,128,0,1,float16,fp8,0,0.0784800002972285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,128,0,1,fp8,fp8,0,0.06874133149782817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,128,0,1,float16,float16,0,0.07986133297284444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,128,0,1,float16,fp8,0,0.08096000055472057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,128,0,1,fp8,fp8,0,0.07091733316580455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,128,0,1,float16,float16,0,0.08122666676839192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,128,0,1,float16,fp8,0,0.08230933547019958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,128,0,1,float16,fp8,0,0.05121066669623057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,128,0,1,fp8,fp8,0,0.07509866853555043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,128,0,1,float16,float16,0,0.053120002150535583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,128,0,1,float16,fp8,0,0.05385066568851471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,128,0,1,fp8,fp8,0,0.051882664362589516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,128,0,1,float16,float16,0,0.051311999559402466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,128,0,1,fp8,fp8,0,0.04470933477083842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,128,0,1,float16,float16,0,0.05014933149019877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,128,0,1,float16,float16,0,0.051167999704678856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,128,0,1,float16,fp8,0,0.0513866643110911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,128,0,1,fp8,fp8,0,0.04525866607824961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,128,0,1,float16,float16,0,0.05067199965318044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,128,0,1,float16,fp8,0,0.05120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,128,0,1,fp8,fp8,0,0.04517866671085358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,128,0,1,float16,fp8,0,0.052005335688591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,128,0,1,fp8,fp8,0,0.04588800172011057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,128,0,1,float16,float16,0,1.965269406636556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,128,0,1,fp8,fp8,0,1.7402186393737793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,128,0,1,float16,fp8,0,1.9466932614644368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,128,0,1,float16,float16,0,1.9604533513387044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,128,0,1,fp8,fp8,0,1.8472320238749187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,128,0,1,float16,fp8,0,1.9620107014973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,128,0,1,float16,float16,0,2.021226723988851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,128,0,1,float16,fp8,0,2.016181310017904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,128,0,1,float16,float16,0,1.2777493000030518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,128,0,1,fp8,fp8,0,1.0909279982248943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,128,0,1,fp8,fp8,0,1.9841173489888508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,128,0,1,float16,fp8,0,1.2445120016733806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,128,0,1,float16,float16,0,2.1868799527486167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,128,0,1,float16,fp8,0,2.127098719278971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,128,0,1,fp8,fp8,0,2.019946734110514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,128,0,1,float16,float16,0,0.9893386363983154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,128,0,1,fp8,fp8,0,0.8886079788208008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,128,0,1,float16,fp8,0,0.9930826822916666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,128,0,1,float16,float16,0,1.000922679901123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,128,0,1,fp8,fp8,0,0.9119893709818522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,128,0,1,float16,fp8,0,1.0011040369669597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,128,0,1,float16,float16,0,1.0249066352844238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,128,0,1,float16,fp8,0,1.0115520159403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,128,0,1,fp8,fp8,0,1.006053368250529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,128,0,1,float16,float16,0,0.6472533146540324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,128,0,1,float16,float16,0,1.1050506432851155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,128,0,1,fp8,fp8,0,1.0216906865437825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,128,0,1,float16,fp8,0,1.0858293374379475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,128,0,1,float16,fp8,0,0.6220266819000244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,128,0,1,fp8,fp8,0,0.5556053320566813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,128,0,1,float16,float16,0,0.511733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,128,0,1,float16,fp8,0,0.5108533302942911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,128,0,1,fp8,fp8,0,0.45216532548268634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,128,0,1,float16,float16,0,0.5125439961751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,128,0,1,float16,fp8,0,0.5166133244832357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,128,0,1,fp8,fp8,0,0.46718935171763104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,128,0,1,float16,float16,0,0.52292267481486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,128,0,1,float16,fp8,0,0.5242666800816854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,128,0,1,fp8,fp8,0,0.5130079984664917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,128,0,1,float16,float16,0,0.5654879808425903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,128,0,1,float16,float16,0,0.3338506619135539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,128,0,1,float16,fp8,0,0.5543200174967448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,128,0,1,fp8,fp8,0,0.517685333887736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,128,0,1,float16,fp8,0,0.3240320086479187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,128,0,1,fp8,fp8,0,0.2858560085296631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,128,0,1,float16,float16,0,0.2688106695810954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,128,0,1,float16,float16,0,0.26945600907007855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,128,0,1,float16,fp8,0,0.2680640021959941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,128,0,1,fp8,fp8,0,0.23534399271011353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,128,0,1,float16,fp8,0,0.269978662331899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,128,0,1,fp8,fp8,0,0.23840532700220743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,128,0,1,float16,float16,0,0.27820799748102826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,128,0,1,float16,fp8,0,0.27410133679707843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,128,0,1,fp8,fp8,0,0.2632799943288167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,128,0,1,float16,float16,0,0.2945546706517537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,128,0,1,float16,fp8,0,0.2921760082244873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,128,0,1,fp8,fp8,0,0.2645866672197978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,128,0,1,float16,fp8,0,0.14710399508476257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,128,0,1,fp8,fp8,0,0.11512533823649089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,128,0,1,float16,float16,0,0.1804479956626892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,128,0,1,float16,fp8,0,0.17720532417297363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,128,0,1,fp8,fp8,0,0.14652799566586813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,128,0,1,float16,float16,0,0.14702399571736655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,128,0,1,float16,float16,0,0.14963199694951376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,128,0,1,float16,fp8,0,0.1474186678727468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,128,0,1,fp8,fp8,0,0.11870400110880534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,128,0,1,float16,float16,0,0.15122133493423462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,128,0,1,float16,float16,0,0.0962666670481364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,128,0,1,float16,fp8,0,0.1502026617527008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,128,0,1,fp8,fp8,0,0.12577600280443826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,128,0,1,float16,float16,0,0.16035733620325723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,128,0,1,float16,fp8,0,0.1584106683731079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,128,0,1,fp8,fp8,0,0.13004799683888754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,128,0,1,float16,fp8,0,0.09438400467236836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,128,0,1,fp8,fp8,0,0.08146133522192638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,128,0,1,float16,float16,0,0.07619733115037282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,128,0,1,float16,fp8,0,0.0761706680059433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,128,0,1,fp8,fp8,0,0.06305066744486491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,128,0,1,float16,float16,0,0.07755733529726665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,128,0,1,float16,fp8,0,0.07732800145943959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,128,0,1,fp8,fp8,0,0.0658240020275116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,128,0,1,float16,float16,0,0.07964266836643219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,128,0,1,float16,fp8,0,0.08031466603279114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,128,0,1,fp8,fp8,0,0.06946666538715363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,128,0,1,float16,float16,0,0.04318933188915253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,128,0,1,float16,float16,0,0.08244266609350841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,128,0,1,float16,fp8,0,0.08156266808509827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,128,0,1,fp8,fp8,0,0.07117866476376851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,128,0,1,float16,float16,0,0.05053866902987162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,128,0,1,float16,fp8,0,0.04916800061861674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,128,0,1,fp8,fp8,0,0.045642669002215065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,128,0,1,float16,fp8,0,0.043194666504859924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,128,0,1,fp8,fp8,0,0.036981334288915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,128,0,1,float16,float16,0,0.044138665000597634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,128,0,1,float16,fp8,0,0.043477331598599754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,128,0,1,fp8,fp8,0,0.03766400118668874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,128,0,1,float16,float16,0,0.04484800000985464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,128,0,1,float16,fp8,0,0.044213334719340004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,128,0,1,fp8,fp8,0,0.03886399914820989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,128,0,1,float16,float16,0,0.045663997530937195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,128,0,1,float16,fp8,0,0.04533866544564565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,128,0,1,fp8,fp8,0,0.04121600091457367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,128,0,1,float16,float16,0,0.03426666557788849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,128,0,1,float16,fp8,0,0.03425066669782003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,128,0,1,fp8,fp8,0,0.02914133419593175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,128,0,1,float16,float16,0,0.032144000132878624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,128,0,1,float16,fp8,0,0.032261334359645844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,128,0,1,fp8,fp8,0,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,128,0,1,float16,float16,0,0.032831999162832894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,128,0,1,float16,fp8,0,0.03288000077009201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,128,0,1,fp8,fp8,0,0.02787200113137563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,128,0,1,float16,float16,0,0.03339199970165888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,128,0,1,float16,fp8,0,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,128,0,1,fp8,fp8,0,0.028944000601768494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,128,0,1,float16,float16,0,0.03386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,128,0,1,float16,fp8,0,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,128,0,1,fp8,fp8,0,0.029098667204380035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,128,0,1,float16,float16,0,1.5319786071777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,128,0,1,fp8,fp8,0,1.442815939585368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,128,0,1,float16,fp8,0,1.5275840759277344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,128,0,1,float16,float16,0,1.546592076619466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,128,0,1,float16,fp8,0,1.5361919403076172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,128,0,1,fp8,fp8,0,1.5331679979960124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,128,0,1,float16,float16,0,1.5969759623209636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,128,0,1,float16,fp8,0,1.5800800323486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,128,0,1,fp8,fp8,0,1.6294347445170085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,128,0,1,float16,float16,0,1.7776586214701335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,128,0,1,float16,float16,0,1.0660266876220703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,128,0,1,float16,fp8,0,1.0163946946461995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,128,0,1,float16,fp8,0,1.7058879534403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,128,0,1,fp8,fp8,0,1.66976531346639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,128,0,1,float16,float16,0,0.7693066596984863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,128,0,1,fp8,fp8,0,0.9247466723124186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,128,0,1,float16,fp8,0,0.7785546779632568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,128,0,1,fp8,fp8,0,0.7125120162963867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,128,0,1,float16,float16,0,0.7781279881795248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,128,0,1,float16,fp8,0,0.7758346398671468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,128,0,1,fp8,fp8,0,0.7515199979146322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,128,0,1,float16,float16,0,0.8068479696909586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,128,0,1,float16,fp8,0,0.804207960764567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,128,0,1,fp8,fp8,0,0.8337492942810059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,128,0,1,float16,float16,0,0.8856746355692545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,128,0,1,float16,float16,0,0.5399519999821981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,128,0,1,float16,fp8,0,0.8613333702087402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,128,0,1,fp8,fp8,0,0.8523413340250651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,128,0,1,float16,fp8,0,0.5178026755650839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,128,0,1,fp8,fp8,0,0.47146133581797284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,128,0,1,float16,float16,0,0.401583989461263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,128,0,1,fp8,fp8,0,0.36095468203226727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,128,0,1,float16,fp8,0,0.399125337600708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,128,0,1,float16,float16,0,0.4027093251546224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,128,0,1,float16,fp8,0,0.4020853439966838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,128,0,1,fp8,fp8,0,0.3655093510945638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,128,0,1,float16,float16,0,0.418287992477417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,128,0,1,float16,fp8,0,0.445578654607137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,128,0,1,float16,fp8,0,0.4109813372294108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,128,0,1,fp8,fp8,0,0.4225333531697591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,128,0,1,float16,float16,0,0.45466665426890057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,128,0,1,float16,float16,0,0.280511995156606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,128,0,1,fp8,fp8,0,0.43063998222351074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,128,0,1,float16,fp8,0,0.2698826591173808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,128,0,1,fp8,fp8,0,0.24036800861358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,128,0,1,float16,fp8,0,0.21316266059875488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,128,0,1,float16,float16,0,0.21095999081929526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,128,0,1,float16,fp8,0,0.21067200104395548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,128,0,1,fp8,fp8,0,0.1906933387120565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,128,0,1,float16,float16,0,0.21068267027537027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,128,0,1,fp8,fp8,0,0.19532267252604166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,128,0,1,float16,float16,0,0.21738133827845255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,128,0,1,fp8,fp8,0,0.21911466121673584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,128,0,1,float16,fp8,0,0.14637333154678345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,128,0,1,fp8,fp8,0,0.21735467513402304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,128,0,1,float16,fp8,0,0.21685866514841715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,128,0,1,float16,float16,0,0.23914666970570883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,128,0,1,float16,fp8,0,0.23455999294916788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,128,0,1,float16,float16,0,0.11710932850837708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,128,0,1,float16,float16,0,0.15011200308799744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,128,0,1,fp8,fp8,0,0.1268160045146942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,128,0,1,float16,float16,0,0.11499200264612834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,128,0,1,float16,fp8,0,0.11476266384124756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,128,0,1,fp8,fp8,0,0.09810133775075276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,128,0,1,float16,fp8,0,0.11692266662915547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,128,0,1,fp8,fp8,0,0.09815999865531921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,128,0,1,float16,fp8,0,0.12778133153915405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,128,0,1,float16,float16,0,0.1223360002040863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,128,0,1,float16,fp8,0,0.12064533432324727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,128,0,1,fp8,fp8,0,0.10532800356547038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,128,0,1,float16,float16,0,0.1285653313000997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,128,0,1,fp8,fp8,0,0.10787733395894368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,128,0,1,float16,float16,0,0.08559999863306682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,128,0,1,float16,fp8,0,0.08257600168387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,128,0,1,fp8,fp8,0,0.07122666637102763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,128,0,1,fp8,fp8,0,0.056618665655454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,128,0,1,float16,float16,0,0.06227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,128,0,1,float16,fp8,0,0.061706667145093284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,128,0,1,fp8,fp8,0,0.05454400181770325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,128,0,1,float16,float16,0,0.06333866715431213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,128,0,1,float16,fp8,0,0.06347199777762096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,128,0,1,float16,float16,0,0.06673599779605865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,128,0,1,float16,float16,0,0.043290664752324425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,128,0,1,float16,fp8,0,0.06689600149790446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,128,0,1,fp8,fp8,0,0.059749335050582886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,128,0,1,float16,float16,0,0.06911466519037883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,128,0,1,float16,fp8,0,0.06785599887371063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,128,0,1,fp8,fp8,0,0.061066667238871254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,128,0,1,float16,fp8,0,0.04215999941031138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,128,0,1,fp8,fp8,0,0.04197866717974345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,128,0,1,float16,float16,0,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,128,0,1,float16,fp8,0,0.03714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,128,0,1,fp8,fp8,0,0.031658666829268135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,128,0,1,float16,float16,0,0.037658666570981346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,128,0,1,float16,fp8,0,0.03722133239110311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,128,0,1,fp8,fp8,0,0.03233066697915395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,128,0,1,float16,float16,0,0.038762666285037994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,128,0,1,float16,fp8,0,0.03882666677236557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,128,0,1,float16,float16,0,0.027690666417280834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,128,0,1,float16,fp8,0,0.026975999275843304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,128,0,1,fp8,fp8,0,0.037248000502586365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,128,0,1,float16,float16,0,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,128,0,1,float16,fp8,0,0.03851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,128,0,1,fp8,fp8,0,0.0367253323396047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,128,0,1,fp8,fp8,0,0.025008000433444977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,128,0,1,float16,float16,0,0.026047999660174053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,128,0,1,float16,fp8,0,0.025792000194390614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,128,0,1,fp8,fp8,0,0.022976001103719074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,128,0,1,float16,float16,0,0.02643200010061264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,128,0,1,float16,fp8,0,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,128,0,1,fp8,fp8,0,0.023957334458827972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,128,0,1,float16,float16,0,0.026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,128,0,1,float16,fp8,0,0.026906666656335194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,128,0,1,fp8,fp8,0,0.02456533412138621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,128,0,1,float16,float16,0,0.027322667340437572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,128,0,1,float16,fp8,0,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,128,0,1,float16,float16,0,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,128,0,1,float16,fp8,0,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,128,0,1,fp8,fp8,0,0.024405332903067272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,128,0,1,float16,float16,0,0.0240639994541804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,128,0,1,float16,fp8,0,0.023930666347344715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,128,0,1,fp8,fp8,0,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,128,0,1,fp8,fp8,0,0.020469332734743755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,128,0,1,float16,float16,0,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,128,0,1,float16,fp8,0,0.023029332359631855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,128,0,1,fp8,fp8,0,0.020576000213623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,128,0,1,float16,float16,0,0.023898666103680927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,128,0,1,float16,fp8,0,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,128,0,1,fp8,fp8,0,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,128,0,1,float16,float16,0,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,128,0,1,float16,fp8,0,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,128,0,1,fp8,fp8,0,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,128,0,1,float16,float16,0,0.672538677851359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,128,0,1,float16,fp8,0,0.6714933713277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,128,0,1,fp8,fp8,0,0.5758133331934611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,128,0,1,float16,float16,0,0.6810719966888428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,128,0,1,float16,fp8,0,0.6799360116322836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,128,0,1,fp8,fp8,0,0.6261599858601888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,128,0,1,float16,float16,0,0.7066613038380941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,128,0,1,float16,fp8,0,0.7038186391194662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,128,0,1,fp8,fp8,0,0.7011786301930746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,128,0,1,float16,float16,0,0.7865813573201498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,128,0,1,float16,fp8,0,0.762938658396403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,128,0,1,float16,float16,0,0.5023200114568075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,128,0,1,float16,fp8,0,0.47658665974934894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,128,0,1,fp8,fp8,0,0.7127573490142822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,128,0,1,fp8,fp8,0,0.40426135063171387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,128,0,1,float16,float16,0,0.3482453425725301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,128,0,1,float16,fp8,0,0.34276799360911053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,128,0,1,fp8,fp8,0,0.2898506720860799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,128,0,1,float16,float16,0,0.35028799374898273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,128,0,1,float16,fp8,0,0.3509440024693807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,128,0,1,fp8,fp8,0,0.30959467093149823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,128,0,1,float16,fp8,0,0.35972265402475995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,128,0,1,float16,float16,0,0.36289068063100177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,128,0,1,fp8,fp8,0,0.3596479892730713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,128,0,1,float16,float16,0,0.3975946505864461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,128,0,1,float16,fp8,0,0.3879573345184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,128,0,1,float16,float16,0,0.25965867439905804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,128,0,1,fp8,fp8,0,0.36530665556589764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,128,0,1,fp8,fp8,0,0.21154133478800455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,128,0,1,float16,fp8,0,0.24848000208536783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,128,0,1,float16,float16,0,0.18185599644978842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,128,0,1,float16,fp8,0,0.18194133043289185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,128,0,1,fp8,fp8,0,0.1548640032609304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,128,0,1,float16,float16,0,0.1916853388150533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,128,0,1,float16,float16,0,0.18450667460759482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,128,0,1,float16,fp8,0,0.1832053263982137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,128,0,1,fp8,fp8,0,0.15959466497103372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,128,0,1,float16,fp8,0,0.1891040007273356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,128,0,1,fp8,fp8,0,0.18387200435002646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,128,0,1,float16,float16,0,0.2099306583404541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,128,0,1,float16,fp8,0,0.20731733242670694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,128,0,1,fp8,fp8,0,0.18573333819707236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,128,0,1,float16,float16,0,0.14102933804194132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,128,0,1,float16,fp8,0,0.13566933075586954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,128,0,1,fp8,fp8,0,0.1190826694170634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,128,0,1,float16,float16,0,0.10040533542633057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,128,0,1,fp8,fp8,0,0.08907199899355571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,128,0,1,float16,fp8,0,0.10128532846768697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,128,0,1,fp8,fp8,0,0.08712533116340637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,128,0,1,float16,float16,0,0.1037600040435791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,128,0,1,float16,fp8,0,0.10293333729108174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,128,0,1,float16,float16,0,0.1076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,128,0,1,float16,fp8,0,0.10690133770306905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,128,0,1,fp8,fp8,0,0.09737599889437358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,128,0,1,float16,float16,0,0.1169546643892924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,128,0,1,float16,fp8,0,0.11317867040634155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,128,0,1,fp8,fp8,0,0.09798399607340495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,128,0,1,float16,float16,0,0.08181866506735484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,128,0,1,float16,fp8,0,0.0783786674340566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,128,0,1,float16,fp8,0,0.057722667853037514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,128,0,1,fp8,fp8,0,0.06554133196671803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,128,0,1,float16,float16,0,0.05741333464781443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,128,0,1,float16,fp8,0,0.056517332792282104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,128,0,1,fp8,fp8,0,0.048725331823031105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,128,0,1,float16,float16,0,0.05764799813429514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,128,0,1,fp8,fp8,0,0.050474668542544045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,128,0,1,float16,float16,0,0.0613919993241628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,128,0,1,float16,fp8,0,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,128,0,1,fp8,fp8,0,0.05487466851870219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,128,0,1,float16,float16,0,0.06542933483918507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,128,0,1,float16,fp8,0,0.063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,128,0,1,fp8,fp8,0,0.058058664202690125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,128,0,1,float16,float16,0,0.04019733270009359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,128,0,1,float16,fp8,0,0.038986665507157646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,128,0,1,fp8,fp8,0,0.039221333960692085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,128,0,1,float16,float16,0,0.03365333378314972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,128,0,1,float16,fp8,0,0.034741332133611046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,128,0,1,fp8,fp8,0,0.029472000896930695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,128,0,1,float16,float16,0,0.03454400102297465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,128,0,1,float16,fp8,0,0.03431999931732813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,128,0,1,fp8,fp8,0,0.030778666337331135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,128,0,1,float16,float16,0,0.035743998984495796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,128,0,1,float16,fp8,0,0.035205334424972534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,128,0,1,fp8,fp8,0,0.03310399999221166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,128,0,1,float16,float16,0,0.03588266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,128,0,1,float16,fp8,0,0.03566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,128,0,1,fp8,fp8,0,0.03421333432197571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,128,0,1,float16,float16,0,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,128,0,1,float16,fp8,0,0.024901332954565685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,128,0,1,fp8,fp8,0,0.022986667851607006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,128,0,1,float16,float16,0,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,128,0,1,float16,fp8,0,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,128,0,1,fp8,fp8,0,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,128,0,1,float16,float16,0,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,128,0,1,float16,fp8,0,0.023541333774725597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,128,0,1,fp8,fp8,0,0.02086399992307027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,128,0,1,float16,float16,0,0.024671999116738636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,128,0,1,float16,fp8,0,0.024293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,128,0,1,fp8,fp8,0,0.022085333863894146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,128,0,1,float16,float16,0,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,128,0,1,float16,fp8,0,0.02476799984773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,128,0,1,fp8,fp8,0,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,128,0,1,float16,float16,0,0.020351999749739964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,128,0,1,fp8,fp8,0,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,128,0,1,float16,fp8,0,0.02072000006834666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,128,0,1,fp8,fp8,0,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,128,0,1,float16,float16,0,0.019733333339293797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,128,0,1,float16,fp8,0,0.020053333292404812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,128,0,1,fp8,fp8,0,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,128,0,1,float16,float16,0,0.01987733319401741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,128,0,1,float16,fp8,0,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,128,0,1,float16,fp8,0,0.019573333362738293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,128,0,1,fp8,fp8,0,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,128,0,1,float16,float16,0,0.020213333268960316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,128,0,1,float16,fp8,0,0.020224000016848247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,128,0,1,float16,float16,0,0.020288000504175823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,128,0,1,float16,fp8,0,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,128,0,1,fp8,fp8,0,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,128,0,1,float16,float16,0,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,128,0,1,float16,fp8,0,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,128,0,1,fp8,fp8,0,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,128,0,1,float16,float16,0,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,128,0,1,fp8,fp8,0,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,128,0,1,fp8,fp8,0,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,128,0,1,float16,float16,0,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,128,0,1,float16,fp8,0,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,128,0,1,fp8,fp8,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,128,0,1,float16,float16,0,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,128,0,1,float16,fp8,0,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,128,0,1,float16,float16,0,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,128,0,1,float16,fp8,0,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,128,0,1,fp8,fp8,0,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,128,0,1,float16,float16,0,0.3214346567789714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,128,0,1,float16,fp8,0,0.32043200731277466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,128,0,1,fp8,fp8,0,0.2922079960505168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,128,0,1,float16,float16,0,0.32523733377456665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,128,0,1,float16,fp8,0,0.32709866762161255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,128,0,1,float16,fp8,0,0.3364266554514567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,128,0,1,fp8,fp8,0,0.30826133489608765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,128,0,1,float16,float16,0,0.343173344930013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,128,0,1,fp8,fp8,0,0.3543786605199178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,128,0,1,float16,float16,0,0.3887840112050374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,128,0,1,fp8,fp8,0,0.3601119915644328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,128,0,1,float16,fp8,0,0.37434132893880206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,128,0,1,float16,float16,0,0.26420267422993976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,128,0,1,float16,fp8,0,0.25521065791447956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,128,0,1,fp8,fp8,0,0.20659732818603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,128,0,1,float16,float16,0,0.17126399278640747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,128,0,1,float16,fp8,0,0.17058666547139487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,128,0,1,fp8,fp8,0,0.15611732999483743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,128,0,1,float16,float16,0,0.17389865716298422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,128,0,1,float16,fp8,0,0.17303466796875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,128,0,1,fp8,fp8,0,0.16022400061289468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,128,0,1,float16,float16,0,0.18433600664138794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,128,0,1,float16,fp8,0,0.18151466051737467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,128,0,1,fp8,fp8,0,0.1825066606203715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,128,0,1,float16,float16,0,0.2084160049756368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,128,0,1,float16,fp8,0,0.20316267013549805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,128,0,1,fp8,fp8,0,0.1857973337173462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,128,0,1,float16,float16,0,0.14387200276056925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,128,0,1,float16,fp8,0,0.14071466525395712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,128,0,1,fp8,fp8,0,0.11256000399589539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,128,0,1,float16,float16,0,0.09715732932090759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,128,0,1,float16,fp8,0,0.09853333234786987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,128,0,1,float16,fp8,0,0.09752000371615092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,128,0,1,fp8,fp8,0,0.08754666646321614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,128,0,1,float16,float16,0,0.10056533416112264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,128,0,1,fp8,fp8,0,0.09737066427866618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,128,0,1,float16,fp8,0,0.11570666233698527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,128,0,1,float16,float16,0,0.11956800023714702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,128,0,1,fp8,fp8,0,0.08955199519793193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,128,0,1,float16,float16,0,0.10776533683141072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,128,0,1,float16,fp8,0,0.10433600346247356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,128,0,1,fp8,fp8,0,0.09852799773216248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,128,0,1,float16,float16,0,0.07764799892902374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,128,0,1,float16,fp8,0,0.07441066702206929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,128,0,1,fp8,fp8,0,0.06398400167624156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,128,0,1,float16,float16,0,0.05499200026194254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,128,0,1,float16,fp8,0,0.054458667834599815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,128,0,1,fp8,fp8,0,0.04858666658401489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,128,0,1,float16,float16,0,0.05566399792830149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,128,0,1,float16,fp8,0,0.05489600201447805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,128,0,1,fp8,fp8,0,0.05134400228659312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,128,0,1,float16,float16,0,0.05955733358860016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,128,0,1,float16,fp8,0,0.05969599882761637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,128,0,1,fp8,fp8,0,0.055205335219701133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,128,0,1,float16,float16,0,0.062421331803003945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,128,0,1,float16,fp8,0,0.06170133252938589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,128,0,1,fp8,fp8,0,0.055973331133524575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,128,0,1,float16,float16,0,0.039936001102129616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,128,0,1,float16,fp8,0,0.03914133210976919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,128,0,1,fp8,fp8,0,0.0377813329299291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,128,0,1,float16,float16,0,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,128,0,1,fp8,fp8,0,0.029978667696317036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,128,0,1,float16,fp8,0,0.033904001116752625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,128,0,1,fp8,fp8,0,0.029445332785447437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,128,0,1,float16,float16,0,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,128,0,1,float16,fp8,0,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,128,0,1,float16,float16,0,0.03501333296298981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,128,0,1,fp8,fp8,0,0.03267733256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,128,0,1,float16,fp8,0,0.03530666728814443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,128,0,1,float16,float16,0,0.024170666933059692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,128,0,1,fp8,fp8,0,0.03180266668399175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,128,0,1,float16,float16,0,0.0359199990828832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,128,0,1,float16,fp8,0,0.035690667728583016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,128,0,1,fp8,fp8,0,0.02260799954334895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,128,0,1,float16,fp8,0,0.0239680012067159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,128,0,1,float16,fp8,0,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,128,0,1,float16,float16,0,0.02237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,128,0,1,float16,fp8,0,0.022389332453409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,128,0,1,fp8,fp8,0,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,128,0,1,float16,float16,0,0.022783999641736347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,128,0,1,float16,float16,0,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,128,0,1,fp8,fp8,0,0.02080533280968666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,128,0,1,float16,float16,0,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,128,0,1,float16,fp8,0,0.02380799998839696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,128,0,1,fp8,fp8,0,0.021749332547187805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,128,0,1,fp8,fp8,0,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,128,0,1,float16,fp8,0,0.024266667664051056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,128,0,1,fp8,fp8,0,0.022282667458057404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,128,0,1,float16,float16,0,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,128,0,1,float16,fp8,0,0.019744000087181728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,128,0,1,float16,float16,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,128,0,1,float16,fp8,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,128,0,1,fp8,fp8,0,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,128,0,1,float16,float16,0,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,128,0,1,float16,fp8,0,0.018650667121013004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,128,0,1,fp8,fp8,0,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,128,0,1,float16,float16,0,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,128,0,1,float16,fp8,0,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,128,0,1,fp8,fp8,0,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,128,0,1,float16,float16,0,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,128,0,1,float16,fp8,0,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,128,0,1,fp8,fp8,0,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,128,0,1,float16,float16,0,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,128,0,1,float16,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,128,0,1,fp8,fp8,0,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,128,0,1,fp8,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,128,0,1,float16,fp8,0,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,128,0,1,fp8,fp8,0,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,128,0,1,float16,float16,0,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,128,0,1,float16,float16,0,0.016623999923467636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,128,0,1,fp8,fp8,0,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,128,0,1,float16,fp8,0,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,128,0,1,float16,fp8,0,0.016309333344300587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,128,0,1,float16,float16,0,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,128,0,1,float16,float16,0,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,128,0,1,float16,fp8,0,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,128,0,1,fp8,fp8,0,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,128,0,1,float16,fp8,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,128,0,1,float16,float16,0,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,128,0,1,fp8,fp8,0,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,128,0,1,float16,fp8,0,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,128,0,1,fp8,fp8,0,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,128,0,1,float16,float16,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,128,0,1,float16,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,128,0,1,fp8,fp8,0,0.01666133354107539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,128,0,1,float16,float16,0,0.01591466615597407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,128,0,1,float16,fp8,0,0.01613866661985715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,128,0,1,fp8,fp8,0,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,128,0,1,float16,float16,0,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,128,0,1,float16,fp8,0,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,128,0,1,fp8,fp8,0,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,128,0,1,float16,float16,0,0.18986133734385172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,128,0,1,float16,fp8,0,0.1918026606241862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,128,0,1,fp8,fp8,0,0.20866133769353232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,128,0,1,float16,float16,0,0.1940106749534607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,128,0,1,float16,fp8,0,0.1938506762186686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,128,0,1,fp8,fp8,0,0.21420800685882568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,128,0,1,float16,float16,0,0.20670932531356812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,128,0,1,float16,fp8,0,0.20162665843963623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,128,0,1,fp8,fp8,0,0.23448532819747925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,128,0,1,float16,float16,0,0.2218773365020752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,128,0,1,float16,fp8,0,0.21885865926742554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,128,0,1,float16,float16,0,0.14296000202496847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,128,0,1,fp8,fp8,0,0.24051199356714884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,128,0,1,fp8,fp8,0,0.11554666360219319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,128,0,1,float16,fp8,0,0.10353066523869832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,128,0,1,float16,fp8,0,0.13913066188494363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,128,0,1,fp8,fp8,0,0.13926933209101358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,128,0,1,float16,float16,0,0.1037546694278717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,128,0,1,float16,float16,0,0.10570133725802104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,128,0,1,float16,fp8,0,0.10602666934331258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,128,0,1,fp8,fp8,0,0.11727466185887654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,128,0,1,float16,float16,0,0.1143946647644043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,128,0,1,float16,fp8,0,0.11828800042470296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,128,0,1,fp8,fp8,0,0.12718933820724487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,128,0,1,float16,fp8,0,0.11213333408037822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,128,0,1,fp8,fp8,0,0.12503467003504434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,128,0,1,float16,float16,0,0.12083199620246887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,128,0,1,float16,float16,0,0.07762666543324788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,128,0,1,float16,fp8,0,0.07437866429487865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,128,0,1,fp8,fp8,0,0.0772159993648529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,128,0,1,float16,fp8,0,0.05820266902446747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,128,0,1,float16,float16,0,0.05748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,128,0,1,float16,fp8,0,0.05789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,128,0,1,fp8,fp8,0,0.0641653339068095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,128,0,1,float16,float16,0,0.058687999844551086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,128,0,1,fp8,fp8,0,0.06606400012969971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,128,0,1,float16,float16,0,0.060549333691596985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,128,0,1,fp8,fp8,0,0.0708426684141159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,128,0,1,float16,fp8,0,0.060789331793785095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,128,0,1,fp8,fp8,0,0.06954666475454967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,128,0,1,fp8,fp8,0,0.045647998650868736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,128,0,1,float16,float16,0,0.06496533254782359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,128,0,1,fp8,fp8,0,0.03731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,128,0,1,float16,fp8,0,0.0636106679836909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,128,0,1,float16,float16,0,0.040709334115187325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,128,0,1,float16,fp8,0,0.038805333276589714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,128,0,1,float16,float16,0,0.03417599946260452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,128,0,1,float16,fp8,0,0.03594133257865906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,128,0,1,float16,fp8,0,0.03421333432197571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,128,0,1,float16,float16,0,0.034741332133611046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,128,0,1,float16,fp8,0,0.03503466645876566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,128,0,1,fp8,fp8,0,0.03763733307520548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,128,0,1,float16,float16,0,0.03611200054486593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,128,0,1,fp8,fp8,0,0.03959999978542328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,128,0,1,float16,float16,0,0.037392000357309975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,128,0,1,float16,fp8,0,0.03677333394686381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,128,0,1,fp8,fp8,0,0.04102399945259094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,128,0,1,float16,fp8,0,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,128,0,1,fp8,fp8,0,0.026501332720120747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,128,0,1,float16,float16,0,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,128,0,1,float16,fp8,0,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,128,0,1,fp8,fp8,0,0.024522667129834492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,128,0,1,float16,float16,0,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,128,0,1,float16,fp8,0,0.023760000864664715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,128,0,1,fp8,fp8,0,0.02497066557407379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,128,0,1,float16,float16,0,0.024874667326609295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,128,0,1,float16,fp8,0,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,128,0,1,fp8,fp8,0,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,128,0,1,float16,float16,0,0.024693332612514496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,128,0,1,float16,fp8,0,0.024879999458789825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,128,0,1,fp8,fp8,0,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,128,0,1,float16,float16,0,0.018863999595244724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,128,0,1,float16,fp8,0,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,128,0,1,fp8,fp8,0,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,128,0,1,fp8,fp8,0,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,128,0,1,float16,float16,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,128,0,1,float16,fp8,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,128,0,1,fp8,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,128,0,1,float16,float16,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,128,0,1,float16,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,128,0,1,float16,float16,0,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,128,0,1,float16,fp8,0,0.01860800012946129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,128,0,1,fp8,fp8,0,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,128,0,1,float16,float16,0,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,128,0,1,float16,float16,0,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,128,0,1,float16,fp8,0,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,128,0,1,fp8,fp8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,128,0,1,float16,float16,0,0.015909332782030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,128,0,1,float16,fp8,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,128,0,1,fp8,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,128,0,1,float16,fp8,0,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,128,0,1,float16,fp8,0,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,128,0,1,fp8,fp8,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,128,0,1,float16,float16,0,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,128,0,1,float16,fp8,0,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,128,0,1,fp8,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,128,0,1,float16,float16,0,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,128,0,1,fp8,fp8,0,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,128,0,1,float16,float16,0,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,128,0,1,float16,fp8,0,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,128,0,1,fp8,fp8,0,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,128,0,1,float16,float16,0,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,128,0,1,float16,fp8,0,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,128,0,1,fp8,fp8,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,128,0,1,float16,float16,0,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,128,0,1,float16,fp8,0,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,128,0,1,fp8,fp8,0,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,128,0,1,float16,float16,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,128,0,1,float16,fp8,0,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,128,0,1,fp8,fp8,0,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,128,0,1,float16,float16,0,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,128,0,1,float16,fp8,0,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,128,0,1,fp8,fp8,0,0.016341333587964375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,128,0,1,float16,float16,0,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,128,0,1,float16,fp8,0,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,128,0,1,fp8,fp8,0,0.016250666230916977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,128,0,1,float16,float16,0,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,128,0,1,float16,fp8,0,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,128,0,1,fp8,fp8,0,0.016165333489576977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,128,0,1,float16,float16,0,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,128,0,1,float16,fp8,0,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,128,0,1,fp8,fp8,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,128,0,1,float16,float16,0,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,128,0,1,float16,fp8,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,128,0,1,float16,float16,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,128,0,1,fp8,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,128,0,1,float16,float16,0,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,128,0,1,float16,float16,0,0.13471466302871704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,128,0,1,float16,fp8,0,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,128,0,1,fp8,fp8,0,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,128,0,1,float16,fp8,0,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,128,0,1,fp8,fp8,0,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,128,0,1,float16,fp8,0,0.13743999600410461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,128,0,1,float16,fp8,0,0.13314132889111838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,128,0,1,fp8,fp8,0,0.16947199900945029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,128,0,1,float16,float16,0,0.13795199990272522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,128,0,1,fp8,fp8,0,0.17128533124923706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,128,0,1,float16,float16,0,0.14321600397427878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,128,0,1,float16,fp8,0,0.14291200041770935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,128,0,1,float16,float16,0,0.15363199512163797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,128,0,1,fp8,fp8,0,0.17919466892878214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,128,0,1,float16,fp8,0,0.14920533696810404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,128,0,1,fp8,fp8,0,0.18121600151062012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,128,0,1,float16,float16,0,0.09246399998664856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,128,0,1,float16,float16,0,0.07271466652552287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,128,0,1,float16,fp8,0,0.09177066882451375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,128,0,1,fp8,fp8,0,0.10609599947929382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,128,0,1,float16,float16,0,0.07205333312352498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,128,0,1,float16,fp8,0,0.07252799967924754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,128,0,1,fp8,fp8,0,0.09073600172996521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,128,0,1,float16,float16,0,0.08104533453782399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,128,0,1,float16,fp8,0,0.07216533521811168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,128,0,1,fp8,fp8,0,0.09281599521636963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,128,0,1,float16,float16,0,0.07593599955240886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,128,0,1,float16,fp8,0,0.07643199960390727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,128,0,1,fp8,fp8,0,0.09760000308354695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,128,0,1,float16,fp8,0,0.08072533210118611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,128,0,1,float16,float16,0,0.04797333478927612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,128,0,1,fp8,fp8,0,0.09942400455474854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,128,0,1,float16,fp8,0,0.046853333711624146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,128,0,1,fp8,fp8,0,0.06050133208433787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,128,0,1,float16,float16,0,0.041162667175134025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,128,0,1,float16,fp8,0,0.04141866664091746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,128,0,1,fp8,fp8,0,0.052245333790779114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,128,0,1,float16,float16,0,0.041722665230433144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,128,0,1,float16,fp8,0,0.041082667807737984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,128,0,1,fp8,fp8,0,0.0525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,128,0,1,float16,float16,0,0.04188266893227895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,128,0,1,float16,fp8,0,0.042597333590189614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,128,0,1,fp8,fp8,0,0.054229333996772766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,128,0,1,float16,float16,0,0.04359466830889384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,128,0,1,float16,fp8,0,0.04318933188915253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,128,0,1,fp8,fp8,0,0.05499200026194254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,128,0,1,fp8,fp8,0,0.032287999987602234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,128,0,1,float16,float16,0,0.028207999964555103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,128,0,1,float16,fp8,0,0.028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,128,0,1,fp8,fp8,0,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,128,0,1,float16,float16,0,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,128,0,1,float16,float16,0,0.02624533325433731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,128,0,1,float16,fp8,0,0.026320000489552815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,128,0,1,float16,float16,0,0.026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,128,0,1,float16,fp8,0,0.026757332185904186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,128,0,1,fp8,fp8,0,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,128,0,1,float16,fp8,0,0.027290667096773785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,128,0,1,fp8,fp8,0,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,128,0,1,float16,float16,0,0.027952000498771667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,128,0,1,float16,fp8,0,0.02826133370399475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,128,0,1,fp8,fp8,0,0.033376000821590424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,128,0,1,float16,float16,0,0.020117333779732387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,128,0,1,float16,fp8,0,0.019909333437681198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,128,0,1,fp8,fp8,0,0.023029332359631855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,128,0,1,float16,float16,0,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,128,0,1,float16,fp8,0,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,128,0,1,fp8,fp8,0,0.02183466653029124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,128,0,1,float16,float16,0,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,128,0,1,float16,fp8,0,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,128,0,1,fp8,fp8,0,0.022096000611782074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,128,0,1,float16,float16,0,0.020186666399240494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,128,0,1,float16,fp8,0,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,128,0,1,fp8,fp8,0,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,128,0,1,float16,fp8,0,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,128,0,1,float16,float16,0,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,128,0,1,float16,fp8,0,0.019685332973798115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,128,0,1,fp8,fp8,0,0.022917332748572033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,128,0,1,float16,float16,0,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,128,0,1,fp8,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,128,0,1,float16,float16,0,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,128,0,1,float16,fp8,0,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,128,0,1,fp8,fp8,0,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,128,0,1,float16,fp8,0,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,128,0,1,float16,float16,0,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,128,0,1,float16,fp8,0,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,128,0,1,fp8,fp8,0,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,128,0,1,float16,float16,0,0.01571200042963028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,128,0,1,fp8,fp8,0,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,128,0,1,float16,float16,0,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,128,0,1,float16,fp8,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,128,0,1,fp8,fp8,0,0.018058666338523228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,128,0,1,float16,float16,0,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,128,0,1,float16,fp8,0,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,128,0,1,fp8,fp8,0,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,128,0,1,float16,float16,0,0.014469332993030548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,128,0,1,float16,fp8,0,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,128,0,1,fp8,fp8,0,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,128,0,1,float16,fp8,0,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,128,0,1,float16,float16,0,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,128,0,1,float16,float16,0,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,128,0,1,float16,fp8,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,128,0,1,fp8,fp8,0,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,128,0,1,float16,float16,0,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,128,0,1,fp8,fp8,0,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,128,0,1,float16,fp8,0,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,128,0,1,fp8,fp8,0,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,128,0,1,float16,float16,0,0.014485333114862442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,128,0,1,float16,fp8,0,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,128,0,1,fp8,fp8,0,0.016693333784739178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,128,0,1,float16,float16,0,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,128,0,1,float16,fp8,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,128,0,1,fp8,fp8,0,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,128,0,1,float16,fp8,0,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,128,0,1,fp8,fp8,0,0.016058667252461117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,128,0,1,float16,float16,0,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,128,0,1,float16,fp8,0,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,128,0,1,fp8,fp8,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,128,0,1,float16,float16,0,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,128,0,1,float16,fp8,0,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,128,0,1,fp8,fp8,0,0.016282666474580765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,128,0,1,float16,float16,0,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,128,0,1,fp8,fp8,0,0.016480000068744022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,128,0,1,float16,fp8,0,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,128,0,1,fp8,fp8,0,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,128,0,1,float16,float16,0,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,128,0,1,float16,float16,0,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,128,0,1,float16,fp8,0,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,128,0,1,float16,float16,0,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,128,0,1,float16,fp8,0,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,128,0,1,fp8,fp8,0,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,128,0,1,float16,fp8,0,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,128,0,1,fp8,fp8,0,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,128,0,1,float16,float16,0,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,128,0,1,float16,fp8,0,0.014490666488806406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,128,0,1,fp8,fp8,0,0.016528000434239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,128,0,1,float16,fp8,0,0.10697600245475769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,128,0,1,float16,float16,0,0.11136533816655476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,128,0,1,float16,float16,0,0.10783466696739197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,128,0,1,fp8,fp8,0,0.1464959979057312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,128,0,1,float16,float16,0,0.10833600163459778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,128,0,1,float16,fp8,0,0.10698133707046509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,128,0,1,fp8,fp8,0,0.14895466963450113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,128,0,1,float16,fp8,0,0.11030399799346924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,128,0,1,fp8,fp8,0,0.15242666999499002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,128,0,1,float16,float16,0,0.11573867003122966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,128,0,1,float16,fp8,0,0.11384532848993938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,128,0,1,float16,float16,0,0.06683200101057689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,128,0,1,fp8,fp8,0,0.15412267049153647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,128,0,1,float16,fp8,0,0.06453333298365276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,128,0,1,fp8,fp8,0,0.08898666501045227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,128,0,1,float16,float16,0,0.059263999263445534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,128,0,1,float16,fp8,0,0.05879466732343038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,128,0,1,fp8,fp8,0,0.07949866851170857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,128,0,1,float16,float16,0,0.058933332562446594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,128,0,1,float16,fp8,0,0.059674665331840515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,128,0,1,fp8,fp8,0,0.08083199958006541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,128,0,1,float16,float16,0,0.061018665631612144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,128,0,1,float16,fp8,0,0.059664001067479454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,128,0,1,fp8,fp8,0,0.08260799944400787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,128,0,1,fp8,fp8,0,0.04841066896915436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,128,0,1,float16,float16,0,0.06136533121267954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,128,0,1,float16,fp8,0,0.06131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,128,0,1,fp8,fp8,0,0.08477333188056946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,128,0,1,float16,float16,0,0.037263999382654824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,128,0,1,float16,float16,0,0.03580799947182337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,128,0,1,fp8,fp8,0,0.04701333244641622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,128,0,1,float16,fp8,0,0.037258667250474296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,128,0,1,float16,float16,0,0.035258665680885315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,128,0,1,float16,fp8,0,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,128,0,1,fp8,fp8,0,0.04664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,128,0,1,float16,fp8,0,0.03628266602754593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,128,0,1,float16,float16,0,0.036848001182079315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,128,0,1,float16,fp8,0,0.036346666514873505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,128,0,1,fp8,fp8,0,0.04764266808827718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,128,0,1,float16,float16,0,0.036858665446440377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,128,0,1,float16,fp8,0,0.03682133307059606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,128,0,1,fp8,fp8,0,0.04786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,128,0,1,float16,float16,0,0.024501333634058636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,128,0,1,float16,fp8,0,0.025050667424996693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,128,0,1,fp8,fp8,0,0.030640001098314922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,128,0,1,float16,float16,0,0.024069334069887798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,128,0,1,float16,fp8,0,0.024160000185171764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,128,0,1,fp8,fp8,0,0.02941333254178365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,128,0,1,float16,float16,0,0.02402133246262868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,128,0,1,float16,fp8,0,0.024271999796231587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,128,0,1,fp8,fp8,0,0.029605334003766377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,128,0,1,float16,float16,0,0.02407466620206833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,128,0,1,float16,fp8,0,0.024495999018351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,128,0,1,fp8,fp8,0,0.030245333909988403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,128,0,1,float16,float16,0,0.02460266649723053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,128,0,1,float16,fp8,0,0.024826665719350178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,128,0,1,fp8,fp8,0,0.030602666238943737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,128,0,1,float16,fp8,0,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,128,0,1,fp8,fp8,0,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,128,0,1,float16,float16,0,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,128,0,1,float16,fp8,0,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,128,0,1,float16,float16,0,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,128,0,1,fp8,fp8,0,0.020879998803138733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,128,0,1,float16,float16,0,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,128,0,1,float16,fp8,0,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,128,0,1,fp8,fp8,0,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,128,0,1,float16,fp8,0,0.01817600056529045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,128,0,1,fp8,fp8,0,0.022128000855445862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,128,0,1,float16,float16,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,128,0,1,float16,fp8,0,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,128,0,1,float16,fp8,0,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,128,0,1,fp8,fp8,0,0.021781332790851593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,128,0,1,float16,float16,0,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,128,0,1,float16,fp8,0,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,128,0,1,fp8,fp8,0,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,128,0,1,float16,float16,0,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,128,0,1,fp8,fp8,0,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,128,0,1,float16,float16,0,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,128,0,1,float16,fp8,0,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,128,0,1,fp8,fp8,0,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,128,0,1,float16,float16,0,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,128,0,1,float16,fp8,0,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,128,0,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,128,0,1,float16,float16,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,128,0,1,float16,fp8,0,0.01595199977358182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,128,0,1,fp8,fp8,0,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,128,0,1,fp8,fp8,0,0.015989333391189575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,128,0,1,float16,float16,0,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,128,0,1,float16,fp8,0,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,128,0,1,fp8,fp8,0,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,128,0,1,float16,float16,0,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,128,0,1,float16,fp8,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,128,0,1,float16,float16,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,128,0,1,float16,fp8,0,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,128,0,1,fp8,fp8,0,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,128,0,1,float16,float16,0,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,128,0,1,float16,fp8,0,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,128,0,1,fp8,fp8,0,0.016810666769742966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,128,0,1,float16,float16,0,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,128,0,1,float16,fp8,0,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,128,0,1,fp8,fp8,0,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,128,0,1,float16,float16,0,0.014229333649079004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,128,0,1,float16,fp8,0,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,128,0,1,fp8,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,128,0,1,fp8,fp8,0,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,128,0,1,float16,float16,0,0.01369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,128,0,1,float16,fp8,0,0.014122666170199713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,128,0,1,fp8,fp8,0,0.01613866661985715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,128,0,1,float16,fp8,0,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,128,0,1,float16,float16,0,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,128,0,1,float16,fp8,0,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,128,0,1,float16,float16,0,0.014229333649079004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,128,0,1,float16,fp8,0,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,128,0,1,fp8,fp8,0,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,128,0,1,float16,float16,0,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,128,0,1,fp8,fp8,0,0.01624533285697301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,128,0,1,float16,float16,0,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,128,0,1,float16,float16,0,0.013690666606028875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,128,0,1,float16,fp8,0,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,128,0,1,fp8,fp8,0,0.016016000260909397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,128,0,1,float16,float16,0,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,128,0,1,float16,fp8,0,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,128,0,1,fp8,fp8,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,128,0,1,float16,fp8,0,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,128,0,1,fp8,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,128,0,1,float16,float16,0,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,128,0,1,float16,fp8,0,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,128,0,1,fp8,fp8,0,0.016048000504573185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,128,0,1,float16,float16,0,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,128,0,1,float16,fp8,0,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,128,0,1,fp8,fp8,0,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,128,0,1,fp8,fp8,0,8.233162562052408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,128,0,1,fp8,fp8,0,8.272773106892904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,128,0,1,float16,float16,0,12.243940989176432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,128,0,1,float16,fp8,0,12.554740905761719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,128,0,1,float16,float16,0,12.350186665852865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,128,0,1,float16,fp8,0,12.240063985188803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,128,0,1,float16,float16,0,12.3831787109375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,128,0,1,float16,float16,0,6.305898666381836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,128,0,1,fp8,fp8,0,4.3270613352457685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,128,0,1,fp8,fp8,0,8.366997400919596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,128,0,1,float16,fp8,0,6.367957433064778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,128,0,1,fp8,fp8,0,4.1040693918863935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,128,0,1,float16,float16,0,6.263354619344075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,128,0,1,float16,fp8,0,6.219024022420247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,128,0,1,float16,fp8,0,12.511850992838541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,128,0,1,fp8,fp8,0,4.235103925069173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,128,0,1,float16,float16,0,6.638650894165039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,128,0,1,float16,float16,0,3.289519945780436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,128,0,1,fp8,fp8,0,4.212762514750163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,128,0,1,float16,fp8,0,6.8300425211588545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,128,0,1,float16,fp8,0,3.2606026331583657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,128,0,1,float16,float16,0,6.240954717000325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,128,0,1,float16,fp8,0,6.427434921264648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,128,0,1,fp8,fp8,0,2.119935989379883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,128,0,1,fp8,fp8,0,2.0630613962809243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,128,0,1,float16,float16,0,3.178842544555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,128,0,1,float16,fp8,0,3.167114575703939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,128,0,1,float16,float16,0,3.187450726826986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,128,0,1,fp8,fp8,0,2.0786773363749185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,128,0,1,float16,fp8,0,3.1830666859944663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,128,0,1,float16,float16,0,3.1987574895222983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,128,0,1,float16,float16,0,1.7612853050231934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,128,0,1,float16,fp8,0,1.6422346433003743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,128,0,1,float16,fp8,0,3.2634665171305337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,128,0,1,fp8,fp8,0,2.294256051381429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,128,0,1,fp8,fp8,0,1.1467520395914714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,128,0,1,float16,float16,0,1.5898346900939941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,128,0,1,float16,fp8,0,1.7634666760762532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,128,0,1,fp8,fp8,0,1.108896017074585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,128,0,1,fp8,fp8,0,1.0647573471069336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,128,0,1,float16,float16,0,1.7486294110616047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,128,0,1,float16,fp8,0,1.5867946942647297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,128,0,1,fp8,fp8,0,1.106000026067098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,128,0,1,float16,float16,0,1.6243467330932617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,128,0,1,float16,fp8,0,1.6558720270792644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,128,0,1,fp8,fp8,0,5.677434921264648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,128,0,1,fp8,fp8,0,5.101717313130696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,128,0,1,float16,float16,0,7.434250513712565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,128,0,1,float16,fp8,0,7.659215927124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,128,0,1,float16,float16,0,7.436373392740886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,128,0,1,float16,fp8,0,8.416175842285156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,128,0,1,float16,float16,0,7.273199717203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,128,0,1,float16,fp8,0,7.647050857543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,128,0,1,float16,float16,0,3.9897867838541665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,128,0,1,fp8,fp8,0,2.495877265930176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,128,0,1,fp8,fp8,0,5.01142946879069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,128,0,1,float16,fp8,0,3.748138745625814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,128,0,1,fp8,fp8,0,2.372666676839193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,128,0,1,float16,float16,0,3.5928853352864585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,128,0,1,float16,fp8,0,3.6691999435424805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,128,0,1,float16,float16,0,3.986655871073405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,128,0,1,fp8,fp8,0,2.489024003346761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,128,0,1,float16,fp8,0,3.6034507751464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,128,0,1,float16,float16,0,1.9164479573567708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,128,0,1,fp8,fp8,0,2.475466728210449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,128,0,1,float16,fp8,0,1.8290826479593914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,128,0,1,float16,float16,0,3.7321173350016275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,128,0,1,fp8,fp8,0,1.3260266780853271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,128,0,1,float16,fp8,0,3.6507253646850586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,128,0,1,float16,float16,0,1.7761066754659016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,128,0,1,fp8,fp8,0,1.2424373626708984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,128,0,1,float16,fp8,0,1.7708266576131184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,128,0,1,fp8,fp8,0,1.2971839904785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,128,0,1,float16,float16,0,1.8493119875590007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,128,0,1,float16,fp8,0,1.807125409444173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,128,0,1,float16,float16,0,1.8777759869893391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,128,0,1,fp8,fp8,0,1.2877333164215088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,128,0,1,float16,float16,0,0.9725866317749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,128,0,1,fp8,fp8,0,0.6595679918924967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,128,0,1,float16,fp8,0,1.7858452796936035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,128,0,1,float16,fp8,0,1.0320800145467122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,128,0,1,float16,float16,0,0.9655786355336508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,128,0,1,float16,fp8,0,1.0012906392415364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,128,0,1,fp8,fp8,0,0.6981600125630697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,128,0,1,fp8,fp8,0,0.6296373208363851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,128,0,1,float16,float16,0,0.970848004023234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,128,0,1,float16,fp8,0,0.9610026677449545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,128,0,1,float16,float16,0,0.9667680263519287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,128,0,1,fp8,fp8,0,0.6358506679534912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,128,0,1,float16,fp8,0,0.9761013189951578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,128,0,1,fp8,fp8,0,3.5482400258382163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,128,0,1,fp8,fp8,0,3.5042667388916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,128,0,1,float16,float16,0,5.2160905202229815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,128,0,1,float16,fp8,0,5.303429285685222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,128,0,1,float16,float16,0,5.214906692504883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,128,0,1,float16,fp8,0,5.352234522501628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,128,0,1,float16,float16,0,5.186479886372884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,128,0,1,float16,fp8,0,5.089610735575358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,128,0,1,float16,float16,0,2.6449546813964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,128,0,1,fp8,fp8,0,1.843616008758545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,128,0,1,fp8,fp8,0,3.5405279795328775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,128,0,1,float16,fp8,0,2.671130816141764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,128,0,1,fp8,fp8,0,1.6887787183125813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,128,0,1,float16,float16,0,2.4608532587687173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,128,0,1,float16,fp8,0,2.5140906969706216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,128,0,1,float16,float16,0,2.434010664621989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,128,0,1,fp8,fp8,0,1.7113386789957683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,128,0,1,float16,fp8,0,2.5744053522745767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,128,0,1,float16,float16,0,1.3262133598327637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,128,0,1,float16,fp8,0,1.302890698115031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,128,0,1,fp8,fp8,0,1.710192044576009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,128,0,1,float16,float16,0,2.6426560084025064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,128,0,1,fp8,fp8,0,0.9403253396352133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,128,0,1,float16,fp8,0,2.607349395751953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,128,0,1,fp8,fp8,0,0.8904853661855062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,128,0,1,float16,float16,0,1.350661277770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,128,0,1,float16,fp8,0,1.2859679857889812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,128,0,1,float16,float16,0,1.2997066974639893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,128,0,1,fp8,fp8,0,0.9069333076477051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,128,0,1,float16,fp8,0,1.3047146797180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,128,0,1,float16,float16,0,1.3215253353118896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,128,0,1,float16,float16,0,0.7233333587646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,128,0,1,fp8,fp8,0,0.9306026299794515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,128,0,1,fp8,fp8,0,0.4756053288777669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,128,0,1,float16,fp8,0,0.7296960353851318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,128,0,1,float16,fp8,0,1.2946826616923015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,128,0,1,float16,float16,0,0.7169226805369059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,128,0,1,float16,fp8,0,0.7166346708933512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,128,0,1,fp8,fp8,0,0.4556266864140828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,128,0,1,float16,float16,0,0.7186826864878336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,128,0,1,fp8,fp8,0,0.4575093189875285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,128,0,1,float16,fp8,0,0.7110133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,128,0,1,float16,float16,0,0.7029706637064616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,128,0,1,fp8,fp8,0,0.4650719960530599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,128,0,1,float16,fp8,0,0.7192533016204834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,128,0,1,fp8,fp8,0,4.6001278559366865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,128,0,1,fp8,fp8,0,4.55789852142334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,128,0,1,float16,float16,0,6.578490575154622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,128,0,1,float16,fp8,0,6.894512176513672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,128,0,1,float16,float16,0,6.884864171346028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,128,0,1,float16,fp8,0,6.945205052693685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,128,0,1,float16,float16,0,6.933925628662109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,128,0,1,float16,fp8,0,6.844783782958984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,128,0,1,float16,float16,0,3.5725440979003906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,128,0,1,fp8,fp8,0,4.6353333791097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,128,0,1,fp8,fp8,0,2.4170506795247397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,128,0,1,float16,fp8,0,3.5061813990275064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,128,0,1,fp8,fp8,0,2.2765493392944336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,128,0,1,float16,float16,0,3.5687147776285806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,128,0,1,float16,fp8,0,3.1561972300211587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,128,0,1,fp8,fp8,0,2.241077264149984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,128,0,1,float16,float16,0,3.2741705576578775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,128,0,1,float16,fp8,0,3.342543919881185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,128,0,1,float16,float16,0,1.7328853607177734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,128,0,1,float16,fp8,0,1.7658880551656086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,128,0,1,float16,float16,0,3.5720532735188804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,128,0,1,fp8,fp8,0,2.3833440144856772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,128,0,1,fp8,fp8,0,1.1961493492126465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,128,0,1,float16,fp8,0,3.438000043233236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,128,0,1,float16,float16,0,1.600170612335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,128,0,1,float16,fp8,0,1.7035999298095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,128,0,1,fp8,fp8,0,1.1631680329640706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,128,0,1,fp8,fp8,0,1.1381759643554688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,128,0,1,float16,float16,0,1.6289547284444172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,128,0,1,float16,fp8,0,1.6481866836547852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,128,0,1,float16,float16,0,1.632186730702718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,128,0,1,fp8,fp8,0,1.1863146622975667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,128,0,1,float16,fp8,0,1.6279999415079753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,128,0,1,float16,float16,0,0.9128106435139974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,128,0,1,float16,fp8,0,0.8932159741719564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,128,0,1,fp8,fp8,0,0.6203306516011556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,128,0,1,float16,float16,0,0.8769280115763346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,128,0,1,float16,fp8,0,0.8740426699320475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,128,0,1,fp8,fp8,0,0.5685653289159139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,128,0,1,float16,float16,0,0.8643573125203451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,128,0,1,float16,fp8,0,0.8749919732411703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,128,0,1,fp8,fp8,0,0.5807626644770304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,128,0,1,float16,float16,0,0.4393920103708903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,128,0,1,fp8,fp8,0,0.5837546586990356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,128,0,1,float16,fp8,0,0.4381759961446126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,128,0,1,float16,float16,0,0.4227840105692546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,128,0,1,float16,float16,0,0.8691840171813965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,128,0,1,float16,fp8,0,0.42056532700856525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,128,0,1,fp8,fp8,0,0.3307680090268453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,128,0,1,float16,fp8,0,0.8600052992502848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,128,0,1,fp8,fp8,0,0.3173653284708659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,128,0,1,float16,float16,0,0.420357346534729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,128,0,1,fp8,fp8,0,0.31888532638549805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,128,0,1,fp8,fp8,0,0.32184000809987384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,128,0,1,float16,fp8,0,0.4259200096130371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,128,0,1,float16,float16,0,0.42740801970163983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,128,0,1,float16,fp8,0,0.42473065853118896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,128,0,1,fp8,fp8,0,2.6674559911092124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,128,0,1,fp8,fp8,0,2.6679840087890625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,128,0,1,float16,float16,0,3.9671894709269204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,128,0,1,float16,fp8,0,4.151359875996907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,128,0,1,float16,float16,0,3.9792372385660806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,128,0,1,float16,fp8,0,4.007349332173665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,128,0,1,float16,float16,0,4.070357322692871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,128,0,1,float16,fp8,0,4.03554121653239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,128,0,1,float16,float16,0,2.0469013849894204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,128,0,1,fp8,fp8,0,1.5342666308085124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,128,0,1,fp8,fp8,0,2.745386759440104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,128,0,1,float16,fp8,0,2.058490594228109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,128,0,1,float16,float16,0,1.9715360005696614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,128,0,1,float16,fp8,0,1.926469326019287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,128,0,1,fp8,fp8,0,1.4054187138875325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,128,0,1,fp8,fp8,0,1.3635147412618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,128,0,1,float16,float16,0,1.933183987935384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,128,0,1,float16,float16,0,1.0333279768625896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,128,0,1,float16,fp8,0,1.9675626754760742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,128,0,1,float16,float16,0,1.984613259633382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,128,0,1,fp8,fp8,0,1.424191951751709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,128,0,1,float16,fp8,0,1.9982825915018718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,128,0,1,float16,fp8,0,1.0316320260365803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,128,0,1,fp8,fp8,0,0.7703413168589274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,128,0,1,float16,float16,0,0.9677066802978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,128,0,1,float16,fp8,0,0.9988053639729818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,128,0,1,fp8,fp8,0,0.736074686050415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,128,0,1,float16,float16,0,0.9984479745229086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,128,0,1,fp8,fp8,0,0.7248693307240804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,128,0,1,float16,fp8,0,1.0229653517405193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,128,0,1,float16,float16,0,0.9914613564809164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,128,0,1,float16,float16,0,0.5566986799240112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,128,0,1,fp8,fp8,0,0.7278613249460856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,128,0,1,fp8,fp8,0,0.3901439905166626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,128,0,1,float16,fp8,0,0.984549363454183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,128,0,1,float16,fp8,0,0.5547413428624471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,128,0,1,float16,float16,0,0.5553706487019857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,128,0,1,fp8,fp8,0,0.3628373146057129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,128,0,1,float16,fp8,0,0.5292479991912842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,128,0,1,fp8,fp8,0,0.3641706705093384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,128,0,1,float16,float16,0,0.5370399951934814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,128,0,1,float16,fp8,0,0.5343466599782308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,128,0,1,float16,float16,0,0.5397973457972208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,128,0,1,fp8,fp8,0,0.37158934275309247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,128,0,1,fp8,fp8,0,0.21410665909449259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,128,0,1,float16,fp8,0,0.5430986483891805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,128,0,1,float16,float16,0,0.26604266961415607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,128,0,1,float16,float16,0,0.2731253306070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,128,0,1,float16,fp8,0,0.27425066630045575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,128,0,1,float16,fp8,0,0.26665600140889484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,128,0,1,fp8,fp8,0,0.20402665932973227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,128,0,1,float16,float16,0,0.2675679922103882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,128,0,1,fp8,fp8,0,0.20503467321395874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,128,0,1,float16,fp8,0,0.26845866441726685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,128,0,1,float16,float16,0,0.26633065938949585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,128,0,1,float16,fp8,0,0.2677653431892395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,128,0,1,fp8,fp8,0,0.20679465929667154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,128,0,1,fp8,fp8,0,2.665642738342285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,128,0,1,float16,float16,0,3.8874613444010415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,128,0,1,fp8,fp8,0,2.7040319442749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,128,0,1,float16,fp8,0,3.851231892903646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,128,0,1,float16,float16,0,3.8898401260375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,128,0,1,float16,fp8,0,3.8826185862223306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,128,0,1,float16,float16,0,3.9432853062947593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,128,0,1,float16,fp8,0,3.9784905115763345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,128,0,1,float16,float16,0,1.9806507428487141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,128,0,1,fp8,fp8,0,1.4936532974243164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,128,0,1,fp8,fp8,0,2.762282689412435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,128,0,1,float16,fp8,0,2.052677313486735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,128,0,1,float16,float16,0,1.8922719955444336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,128,0,1,fp8,fp8,0,1.3341600100199382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,128,0,1,float16,fp8,0,1.8967572848002117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,128,0,1,float16,float16,0,1.9149227142333984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,128,0,1,fp8,fp8,0,1.3764746983846028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,128,0,1,float16,float16,0,0.9971146583557129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,128,0,1,float16,fp8,0,1.916122595469157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,128,0,1,float16,fp8,0,1.0068106651306152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,128,0,1,fp8,fp8,0,1.3638240496317546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,128,0,1,float16,float16,0,1.8646507263183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,128,0,1,fp8,fp8,0,0.7645653088887533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,128,0,1,float16,fp8,0,1.8993813196818035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,128,0,1,float16,float16,0,0.9324800173441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,128,0,1,fp8,fp8,0,0.7059946854909261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,128,0,1,float16,fp8,0,0.913045326868693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,128,0,1,float16,float16,0,0.9273119767506918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,128,0,1,float16,fp8,0,0.9242880344390869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,128,0,1,fp8,fp8,0,0.7125919659932455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,128,0,1,fp8,fp8,0,0.7343839804331461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,128,0,1,float16,float16,0,0.9325493176778158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,128,0,1,float16,fp8,0,0.9480533599853516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,128,0,1,float16,float16,0,0.5293706655502319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,128,0,1,float16,fp8,0,0.5271519819895426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,128,0,1,fp8,fp8,0,0.39106134573618573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,128,0,1,float16,float16,0,0.4845759868621826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,128,0,1,fp8,fp8,0,0.35582399368286133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,128,0,1,float16,fp8,0,0.497050682703654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,128,0,1,float16,float16,0,0.48948800563812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,128,0,1,float16,fp8,0,0.4985119899113973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,128,0,1,fp8,fp8,0,0.3545866807301839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,128,0,1,fp8,fp8,0,0.3524373372395833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,128,0,1,float16,float16,0,0.5004800160725912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,128,0,1,float16,fp8,0,0.5019840002059937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,128,0,1,float16,float16,0,0.26451200246810913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,128,0,1,fp8,fp8,0,0.20618667205174765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,128,0,1,float16,fp8,0,0.2678933342297872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,128,0,1,float16,float16,0,0.2477333347002665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,128,0,1,float16,fp8,0,0.24872533480326334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,128,0,1,fp8,fp8,0,0.1881813406944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,128,0,1,float16,fp8,0,0.2477653423945109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,128,0,1,float16,float16,0,0.24730134010314941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,128,0,1,fp8,fp8,0,0.19543999433517456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,128,0,1,float16,float16,0,0.24587732553482056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,128,0,1,float16,fp8,0,0.2540213267008464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,128,0,1,fp8,fp8,0,0.19849600394566855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,128,0,1,float16,float16,0,0.1471626659234365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,128,0,1,float16,fp8,0,0.15338133772214255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,128,0,1,fp8,fp8,0,0.12264000376065572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,128,0,1,float16,float16,0,0.1453493336836497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,128,0,1,float16,fp8,0,0.14684266845385233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,128,0,1,fp8,fp8,0,0.11397866408030193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,128,0,1,float16,float16,0,0.14733866850535074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,128,0,1,float16,fp8,0,0.1470186710357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,128,0,1,fp8,fp8,0,0.11591999729474385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,128,0,1,float16,float16,0,0.1465173363685608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,128,0,1,float16,fp8,0,0.14786666631698608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,128,0,1,fp8,fp8,0,0.11909866333007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,128,0,1,fp8,fp8,0,1.6997067133585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,128,0,1,float16,float16,0,2.335301399230957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,128,0,1,float16,fp8,0,2.247594674428304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,128,0,1,fp8,fp8,0,1.7033333778381348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,128,0,1,float16,float16,0,2.2565654118855796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,128,0,1,float16,fp8,0,2.255232016245524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,128,0,1,float16,float16,0,2.3281706174214682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,128,0,1,float16,float16,0,1.2388052940368652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,128,0,1,fp8,fp8,0,0.950650691986084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,128,0,1,float16,fp8,0,1.226261297861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,128,0,1,fp8,fp8,0,1.7940533955891926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,128,0,1,float16,float16,0,1.1291626294453938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,128,0,1,float16,fp8,0,2.390746593475342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,128,0,1,float16,fp8,0,1.1718239784240723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,128,0,1,fp8,fp8,0,0.8617119789123535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,128,0,1,float16,float16,0,1.1269333362579346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,128,0,1,fp8,fp8,0,0.8756159941355387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,128,0,1,float16,fp8,0,1.1323253313700359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,128,0,1,fp8,fp8,0,0.8937973181406657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,128,0,1,float16,float16,0,0.6357440153757731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,128,0,1,float16,float16,0,1.1683200200398762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,128,0,1,float16,fp8,0,1.2018986543019612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,128,0,1,fp8,fp8,0,0.49611198902130127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,128,0,1,float16,fp8,0,0.6337066491444906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,128,0,1,float16,float16,0,0.5733813444773356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,128,0,1,float16,fp8,0,0.5744479894638062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,128,0,1,fp8,fp8,0,0.4463786681493123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,128,0,1,float16,float16,0,0.5778186718622843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,128,0,1,float16,fp8,0,0.5901813507080078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,128,0,1,fp8,fp8,0,0.45477867126464844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,128,0,1,float16,fp8,0,0.5878293514251709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,128,0,1,fp8,fp8,0,0.456496000289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,128,0,1,float16,float16,0,0.5914986530939738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,128,0,1,float16,float16,0,0.34087467193603516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,128,0,1,float16,fp8,0,0.34177064895629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,128,0,1,fp8,fp8,0,0.25756800174713135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,128,0,1,float16,float16,0,0.31514666477839154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,128,0,1,float16,fp8,0,0.3131466706593831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,128,0,1,float16,fp8,0,0.3208426634470622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,128,0,1,fp8,fp8,0,0.2278453310330709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,128,0,1,float16,float16,0,0.31646400690078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,128,0,1,fp8,fp8,0,0.22832000255584717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,128,0,1,float16,float16,0,0.3225546677907308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,128,0,1,float16,fp8,0,0.3274293343226115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,128,0,1,fp8,fp8,0,0.23528534173965454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,128,0,1,float16,fp8,0,0.17439999183019003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,128,0,1,float16,float16,0,0.17192000150680542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,128,0,1,fp8,fp8,0,0.14196800192197165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,128,0,1,float16,float16,0,0.16024000446001688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,128,0,1,fp8,fp8,0,0.1297920048236847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,128,0,1,float16,fp8,0,0.15942399700482687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,128,0,1,fp8,fp8,0,0.12938132882118225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,128,0,1,float16,float16,0,0.1622719963391622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,128,0,1,float16,fp8,0,0.16268799702326456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,128,0,1,float16,float16,0,0.1611786683400472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,128,0,1,float16,fp8,0,0.16476800044377646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,128,0,1,fp8,fp8,0,0.13345066706339517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,128,0,1,float16,float16,0,0.10041067004203796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,128,0,1,float16,float16,0,0.0960746705532074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,128,0,1,float16,fp8,0,0.10108266274134318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,128,0,1,fp8,fp8,0,0.08594133456548055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,128,0,1,float16,float16,0,0.09624532858530681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,128,0,1,float16,fp8,0,0.09696533282597859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,128,0,1,fp8,fp8,0,0.07704000174999237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,128,0,1,float16,fp8,0,0.09700799981753032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,128,0,1,fp8,fp8,0,0.0776693324247996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,128,0,1,float16,float16,0,0.09634133179982503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,128,0,1,float16,fp8,0,0.09738666812578838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,128,0,1,fp8,fp8,0,0.07962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,128,0,1,fp8,fp8,0,1.8098559379577637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,128,0,1,float16,float16,0,2.328575929005941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,128,0,1,float16,fp8,0,2.292912006378174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,128,0,1,fp8,fp8,0,1.806144078572591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,128,0,1,float16,float16,0,2.374704043070475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,128,0,1,float16,fp8,0,2.33733336130778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,128,0,1,float16,float16,0,2.3855573336283364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,128,0,1,float16,fp8,0,2.3482720057169595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,128,0,1,fp8,fp8,0,1.0334186553955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,128,0,1,float16,float16,0,1.3079040050506592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,128,0,1,float16,fp8,0,1.2945386568705242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,128,0,1,float16,float16,0,1.1827200253804524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,128,0,1,fp8,fp8,0,1.8617973327636719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,128,0,1,float16,fp8,0,1.1875840028127034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,128,0,1,fp8,fp8,0,0.9294986724853516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,128,0,1,float16,float16,0,1.182634671529134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,128,0,1,float16,fp8,0,1.1548853715260823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,128,0,1,fp8,fp8,0,0.926421324412028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,128,0,1,float16,float16,0,1.199898640314738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,128,0,1,float16,float16,0,0.6645280122756958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,128,0,1,float16,fp8,0,0.6596906582514445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,128,0,1,fp8,fp8,0,0.9545813401540121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,128,0,1,fp8,fp8,0,0.5379413366317749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,128,0,1,float16,fp8,0,1.1868800322214763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,128,0,1,float16,float16,0,0.5803893407185873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,128,0,1,float16,fp8,0,0.5814986626307169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,128,0,1,fp8,fp8,0,0.4726133346557617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,128,0,1,float16,float16,0,0.583461324373881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,128,0,1,float16,fp8,0,0.5858879884084066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,128,0,1,fp8,fp8,0,0.4750239849090576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,128,0,1,float16,float16,0,0.5979679822921753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,128,0,1,float16,fp8,0,0.6033546527226766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,128,0,1,float16,float16,0,0.3511306842168172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,128,0,1,float16,fp8,0,0.3481173515319824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,128,0,1,fp8,fp8,0,0.48579200108846027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,128,0,1,fp8,fp8,0,0.2863039970397949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,128,0,1,float16,float16,0,0.30872533718744916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,128,0,1,fp8,fp8,0,0.24027733008066812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,128,0,1,float16,fp8,0,0.31125332911809284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,128,0,1,float16,float16,0,0.3139253258705139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,128,0,1,float16,fp8,0,0.3136319915453593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,128,0,1,fp8,fp8,0,0.24041066567103067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,128,0,1,float16,float16,0,0.31942933797836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,128,0,1,float16,fp8,0,0.3192266623179118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,128,0,1,fp8,fp8,0,0.24901866912841797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,128,0,1,float16,float16,0,0.183786670366923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,128,0,1,float16,fp8,0,0.1840320030848185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,128,0,1,fp8,fp8,0,0.1506613294283549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,128,0,1,float16,float16,0,0.1596160034338633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,128,0,1,float16,fp8,0,0.16104533274968466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,128,0,1,fp8,fp8,0,0.13126400113105774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,128,0,1,float16,float16,0,0.1612266699473063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,128,0,1,float16,fp8,0,0.16179200013478598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,128,0,1,fp8,fp8,0,0.13378133376439413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,128,0,1,float16,float16,0,0.16350932916005453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,128,0,1,float16,fp8,0,0.1655306617418925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,128,0,1,fp8,fp8,0,0.1360373298327128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,128,0,1,float16,float16,0,0.09865066409111023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,128,0,1,fp8,fp8,0,0.07498666644096375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,128,0,1,float16,fp8,0,0.09942932923634847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,128,0,1,fp8,fp8,0,0.0867199997107188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,128,0,1,float16,float16,0,0.09091732899347942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,128,0,1,float16,fp8,0,0.09115200241406758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,128,0,1,float16,float16,0,0.09166399637858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,128,0,1,float16,fp8,0,0.09175999959309895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,128,0,1,fp8,fp8,0,0.0764213353395462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,128,0,1,float16,float16,0,0.09239466985066731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,128,0,1,float16,fp8,0,0.09446932872136433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,128,0,1,fp8,fp8,0,0.07965333263079326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,128,0,1,float16,float16,0,0.05942399799823761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,128,0,1,float16,fp8,0,0.06058666606744131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,128,0,1,float16,fp8,0,0.058261334896087646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,128,0,1,fp8,fp8,0,0.05409599840641022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,128,0,1,float16,float16,0,0.057818666100502014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,128,0,1,float16,fp8,0,0.05973333120346069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,128,0,1,fp8,fp8,0,0.04970133304595947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,128,0,1,float16,float16,0,0.057850668827692665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,128,0,1,fp8,fp8,0,0.050213331977526345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,128,0,1,float16,float16,0,0.05807466804981232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,128,0,1,float16,fp8,0,0.05863999823729197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,128,0,1,fp8,fp8,0,0.049695998430252075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,128,0,1,float16,float16,0,1.4608532587687175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,128,0,1,float16,fp8,0,1.4794294039408367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,128,0,1,fp8,fp8,0,1.2031893730163574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,128,0,1,fp8,fp8,0,1.232917308807373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,128,0,1,float16,float16,0,1.4811840057373047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,128,0,1,float16,fp8,0,1.4885120391845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,128,0,1,float16,float16,0,1.5381654103597004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,128,0,1,float16,fp8,0,1.5191359519958496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,128,0,1,float16,float16,0,0.8655786514282227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,128,0,1,fp8,fp8,0,1.2709973653157551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,128,0,1,float16,fp8,0,0.8427466551462809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,128,0,1,fp8,fp8,0,0.7148693402608236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,128,0,1,float16,float16,0,0.73581329981486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,128,0,1,float16,fp8,0,0.7422560056050619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,128,0,1,fp8,fp8,0,0.6197066704432169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,128,0,1,float16,float16,0,0.7506559689839681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,128,0,1,float16,fp8,0,0.7513706684112549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,128,0,1,fp8,fp8,0,0.6223306655883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,128,0,1,float16,float16,0,0.7679146925608317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,128,0,1,float16,float16,0,0.4416693449020386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,128,0,1,float16,fp8,0,0.441429336865743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,128,0,1,float16,fp8,0,0.7640533447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,128,0,1,fp8,fp8,0,0.6494933366775513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,128,0,1,fp8,fp8,0,0.3756320079167684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,128,0,1,float16,float16,0,0.38015464941660565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,128,0,1,float16,float16,0,0.3807786703109741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,128,0,1,float16,fp8,0,0.37903467814127606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,128,0,1,fp8,fp8,0,0.3205759922663371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,128,0,1,float16,fp8,0,0.3868853251139323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,128,0,1,fp8,fp8,0,0.33715200424194336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,128,0,1,fp8,fp8,0,0.32236266136169434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,128,0,1,float16,fp8,0,0.23465599616368613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,128,0,1,float16,float16,0,0.39466134707132977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,128,0,1,float16,fp8,0,0.3967413504918416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,128,0,1,float16,float16,0,0.23609066009521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,128,0,1,fp8,fp8,0,0.19570134083429971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,128,0,1,float16,float16,0,0.20667733748753866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,128,0,1,float16,fp8,0,0.20968000094095865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,128,0,1,float16,fp8,0,0.20814400911331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,128,0,1,fp8,fp8,0,0.16312533617019653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,128,0,1,float16,float16,0,0.2108693321545919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,128,0,1,float16,float16,0,0.20785599946975708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,128,0,1,fp8,fp8,0,0.16591466466585794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,128,0,1,fp8,fp8,0,0.17207467555999756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,128,0,1,float16,fp8,0,0.2156533400217692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,128,0,1,float16,float16,0,0.12556800246238708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,128,0,1,float16,fp8,0,0.12586133678754172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,128,0,1,fp8,fp8,0,0.1077280044555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,128,0,1,float16,float16,0,0.10839466253916423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,128,0,1,float16,fp8,0,0.11116799712181091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,128,0,1,float16,fp8,0,0.109525332848231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,128,0,1,fp8,fp8,0,0.09397332866986592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,128,0,1,float16,float16,0,0.10976533095041911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,128,0,1,fp8,fp8,0,0.09518399834632874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,128,0,1,float16,float16,0,0.11151466766993205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,128,0,1,float16,fp8,0,0.07133333384990692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,128,0,1,float16,float16,0,0.06378133098284404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,128,0,1,float16,fp8,0,0.11395200093587239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,128,0,1,fp8,fp8,0,0.09787733356157939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,128,0,1,float16,float16,0,0.0706879993279775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,128,0,1,fp8,fp8,0,0.06389866769313812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,128,0,1,float16,fp8,0,0.06503466765085857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,128,0,1,fp8,fp8,0,0.054602667689323425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,128,0,1,float16,float16,0,0.064560001095136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,128,0,1,float16,fp8,0,0.06506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,128,0,1,fp8,fp8,0,0.05491200089454651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,128,0,1,float16,float16,0,0.06505066653092702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,128,0,1,float16,fp8,0,0.06579199930032094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,128,0,1,fp8,fp8,0,0.05692266424496969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,128,0,1,float16,float16,0,0.04740799963474274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,128,0,1,float16,fp8,0,0.0483893354733785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,128,0,1,fp8,fp8,0,0.04275733232498169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,128,0,1,float16,float16,0,0.047242666284243263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,128,0,1,float16,fp8,0,0.047423998514811196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,128,0,1,fp8,fp8,0,0.040693332751592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,128,0,1,float16,float16,0,0.046623999873797096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,128,0,1,float16,fp8,0,0.047413334250450134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,128,0,1,fp8,fp8,0,0.04062933226426443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,128,0,1,float16,float16,0,0.04773333172003428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,128,0,1,float16,fp8,0,0.04762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,128,0,1,fp8,fp8,0,0.04167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,128,0,1,float16,float16,0,1.4897440274556477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,128,0,1,float16,fp8,0,1.485381285349528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,128,0,1,fp8,fp8,0,1.3255840142567952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,128,0,1,float16,float16,0,1.508639971415202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,128,0,1,float16,fp8,0,1.496565341949463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,128,0,1,fp8,fp8,0,1.383589267730713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,128,0,1,float16,float16,0,1.5914880434672039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,128,0,1,float16,fp8,0,1.5653600692749023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,128,0,1,float16,float16,0,0.9588692982991537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,128,0,1,float16,fp8,0,0.9294293721516927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,128,0,1,fp8,fp8,0,1.4870826403299968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,128,0,1,float16,float16,0,0.7564427057902018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,128,0,1,fp8,fp8,0,0.8186559677124023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,128,0,1,float16,fp8,0,0.75764266649882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,128,0,1,fp8,fp8,0,0.6630933284759521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,128,0,1,float16,fp8,0,0.7605120340983073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,128,0,1,float16,float16,0,0.7753012975056967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,128,0,1,fp8,fp8,0,0.6862239837646484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,128,0,1,float16,float16,0,0.8033386866251627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,128,0,1,float16,float16,0,0.49076799551645917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,128,0,1,float16,fp8,0,0.7909706433614095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,128,0,1,float16,fp8,0,0.48297067483266193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,128,0,1,fp8,fp8,0,0.4186133146286011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,128,0,1,fp8,fp8,0,0.7491839726765951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,128,0,1,float16,float16,0,0.39819733301798504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,128,0,1,float16,fp8,0,0.39549867312113446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,128,0,1,fp8,fp8,0,0.3415840069452922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,128,0,1,float16,float16,0,0.39585065841674805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,128,0,1,float16,fp8,0,0.3967680136362712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,128,0,1,fp8,fp8,0,0.348906675974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,128,0,1,float16,float16,0,0.41154666741689044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,128,0,1,float16,fp8,0,0.40993599096934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,128,0,1,fp8,fp8,0,0.38973331451416016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,128,0,1,float16,float16,0,0.26038400332132977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,128,0,1,float16,fp8,0,0.2542133331298828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,128,0,1,fp8,fp8,0,0.21676800648371378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,128,0,1,float16,float16,0,0.2089973290761312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,128,0,1,float16,fp8,0,0.21369065841039023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,128,0,1,fp8,fp8,0,0.16492266456286112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,128,0,1,float16,float16,0,0.22378132740656534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,128,0,1,float16,float16,0,0.21261866887410483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,128,0,1,float16,fp8,0,0.2146773338317871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,128,0,1,fp8,fp8,0,0.16727999846140543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,128,0,1,float16,fp8,0,0.14703999956448874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,128,0,1,fp8,fp8,0,0.1123413344224294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,128,0,1,float16,fp8,0,0.21678400039672852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,128,0,1,fp8,fp8,0,0.18452266852060953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,128,0,1,float16,float16,0,0.15012266238530478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,128,0,1,float16,float16,0,0.11061333616574605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,128,0,1,float16,fp8,0,0.11160000165303548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,128,0,1,fp8,fp8,0,0.09016000231107076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,128,0,1,float16,float16,0,0.11165866255760193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,128,0,1,float16,fp8,0,0.11295466621716817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,128,0,1,fp8,fp8,0,0.09132267038027446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,128,0,1,float16,float16,0,0.11691733201344807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,128,0,1,float16,fp8,0,0.11726933717727661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,128,0,1,fp8,fp8,0,0.09870400031407674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,128,0,1,float16,fp8,0,0.060965334375699363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,128,0,1,fp8,fp8,0,0.05097599824269613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,128,0,1,float16,float16,0,0.07289066910743713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,128,0,1,float16,fp8,0,0.07149333258469899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,128,0,1,fp8,fp8,0,0.06348800162474315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,128,0,1,float16,float16,0,0.05950933198134104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,128,0,1,float16,float16,0,0.062021334966023765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,128,0,1,float16,fp8,0,0.0618453323841095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,128,0,1,fp8,fp8,0,0.05203199883302053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,128,0,1,float16,float16,0,0.06474133332570393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,128,0,1,float16,fp8,0,0.064560001095136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,128,0,1,fp8,fp8,0,0.05624533196290334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,128,0,1,float16,float16,0,0.03884266565243403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,128,0,1,float16,fp8,0,0.03809600075085958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,128,0,1,float16,float16,0,0.036805334190527596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,128,0,1,float16,fp8,0,0.03639466563860575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,128,0,1,fp8,fp8,0,0.03544000039498011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,128,0,1,float16,float16,0,0.036357333262761436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,128,0,1,float16,fp8,0,0.036464000741640724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,128,0,1,fp8,fp8,0,0.03178133318821589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,128,0,1,fp8,fp8,0,0.03233066697915395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,128,0,1,float16,fp8,0,0.03791466603676478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,128,0,1,float16,float16,0,0.037861332297325134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,128,0,1,fp8,fp8,0,0.03330666571855545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,128,0,1,float16,float16,0,0.03292799989382426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,128,0,1,float16,fp8,0,0.033701332906881966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,128,0,1,fp8,fp8,0,0.028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,128,0,1,float16,fp8,0,0.03224000086386999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,128,0,1,float16,float16,0,0.03183999905983607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,128,0,1,float16,fp8,0,0.0322773332397143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,128,0,1,fp8,fp8,0,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,128,0,1,float16,float16,0,0.031530665854612984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,128,0,1,fp8,fp8,0,0.027930667002995808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,128,0,1,float16,float16,0,0.03279466678698858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,128,0,1,float16,fp8,0,0.03289599965016047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,128,0,1,fp8,fp8,0,0.02900800108909607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,128,0,1,float16,float16,0,1.1689333120981853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,128,0,1,float16,fp8,0,1.1607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,128,0,1,fp8,fp8,0,1.070853312810262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,128,0,1,float16,float16,0,1.195354700088501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,128,0,1,float16,fp8,0,1.1842453479766846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,128,0,1,fp8,fp8,0,1.151312033335368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,128,0,1,float16,float16,0,1.2511253356933594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,128,0,1,float16,float16,0,0.8052053451538086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,128,0,1,float16,fp8,0,0.7700640360514323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,128,0,1,float16,fp8,0,1.2394026915232341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,128,0,1,fp8,fp8,0,0.69706130027771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,128,0,1,float16,float16,0,0.5980960130691528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,128,0,1,fp8,fp8,0,1.2256960074106853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,128,0,1,float16,fp8,0,0.596010684967041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,128,0,1,fp8,fp8,0,0.5374133189519247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,128,0,1,float16,float16,0,0.605679988861084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,128,0,1,float16,fp8,0,0.6016800006230673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,128,0,1,fp8,fp8,0,0.5690933465957642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,128,0,1,float16,float16,0,0.41211732228597003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,128,0,1,float16,float16,0,0.6375840107599894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,128,0,1,float16,fp8,0,0.6312426726023356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,128,0,1,float16,fp8,0,0.39603201548258465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,128,0,1,fp8,fp8,0,0.6276053190231323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,128,0,1,fp8,fp8,0,0.35393067200978595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,128,0,1,float16,float16,0,0.3073493242263794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,128,0,1,float16,fp8,0,0.30744000275929767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,128,0,1,fp8,fp8,0,0.27405333518981934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,128,0,1,float16,float16,0,0.3131573398907979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,128,0,1,float16,fp8,0,0.31041600306828815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,128,0,1,float16,float16,0,0.21644800901412964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,128,0,1,fp8,fp8,0,0.28309333324432373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,128,0,1,float16,float16,0,0.33246399958928424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,128,0,1,fp8,fp8,0,0.18686934312184653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,128,0,1,float16,fp8,0,0.3232799967130025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,128,0,1,fp8,fp8,0,0.31851200262705487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,128,0,1,float16,fp8,0,0.20989867051442465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,128,0,1,float16,float16,0,0.16209600369135538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,128,0,1,float16,fp8,0,0.16339199741681418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,128,0,1,fp8,fp8,0,0.13633599877357483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,128,0,1,float16,float16,0,0.17485332489013672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,128,0,1,float16,float16,0,0.16920000314712524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,128,0,1,float16,fp8,0,0.16687466700871786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,128,0,1,fp8,fp8,0,0.13876266280810037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,128,0,1,float16,fp8,0,0.17282666762669882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,128,0,1,fp8,fp8,0,0.15421332915623984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,128,0,1,float16,float16,0,0.13134933511416116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,128,0,1,fp8,fp8,0,0.09647466739018758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,128,0,1,float16,fp8,0,0.12868266304334006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,128,0,1,fp8,fp8,0,0.07754666606585185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,128,0,1,float16,float16,0,0.09112000465393066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,128,0,1,float16,fp8,0,0.09041600426038106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,128,0,1,fp8,fp8,0,0.07592533528804779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,128,0,1,float16,float16,0,0.09184533357620239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,128,0,1,float16,fp8,0,0.061978667974472046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,128,0,1,float16,fp8,0,0.09112000465393066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,128,0,1,float16,float16,0,0.0963253378868103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,128,0,1,float16,fp8,0,0.09647466739018758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,128,0,1,fp8,fp8,0,0.08291733264923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,128,0,1,float16,float16,0,0.063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,128,0,1,fp8,fp8,0,0.05769066512584686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,128,0,1,float16,fp8,0,0.05147733290990194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,128,0,1,float16,float16,0,0.0499946673711141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,128,0,1,float16,fp8,0,0.050213331977526345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,128,0,1,fp8,fp8,0,0.04422933359940847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,128,0,1,float16,float16,0,0.05178666611512502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,128,0,1,float16,float16,0,0.03461866577466329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,128,0,1,fp8,fp8,0,0.04667200148105621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,128,0,1,float16,float16,0,0.054570664962132774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,128,0,1,float16,fp8,0,0.05426666637261709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,128,0,1,fp8,fp8,0,0.04842133323351542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,128,0,1,float16,fp8,0,0.03602666656176249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,128,0,1,fp8,fp8,0,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,128,0,1,float16,float16,0,0.031888000667095184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,128,0,1,float16,fp8,0,0.031930667658646904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,128,0,1,fp8,fp8,0,0.029552000264326733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,128,0,1,float16,float16,0,0.03219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,128,0,1,float16,fp8,0,0.03197866678237915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,128,0,1,fp8,fp8,0,0.029909332593282063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,128,0,1,float16,float16,0,0.03339733431736628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,128,0,1,float16,fp8,0,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,128,0,1,fp8,fp8,0,0.030837332208951313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,128,0,1,float16,fp8,0,0.025674665967623394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,128,0,1,float16,float16,0,0.026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,128,0,1,float16,fp8,0,0.026362667481104534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,128,0,1,fp8,fp8,0,0.024698667228221893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,128,0,1,float16,float16,0,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,128,0,1,fp8,fp8,0,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,128,0,1,float16,float16,0,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,128,0,1,float16,fp8,0,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,128,0,1,float16,float16,0,0.02294933299223582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,128,0,1,fp8,fp8,0,0.02383466561635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,128,0,1,float16,float16,0,0.02604266752799352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,128,0,1,float16,fp8,0,0.026250667870044708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,128,0,1,fp8,fp8,0,0.024058667321999867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,128,0,1,float16,fp8,0,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,128,0,1,fp8,fp8,0,0.02059200033545494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,128,0,1,float16,float16,0,0.022778667509555817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,128,0,1,float16,fp8,0,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,128,0,1,fp8,fp8,0,0.01960533360640208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,128,0,1,float16,float16,0,0.022042666872342426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,128,0,1,float16,fp8,0,0.022101332743962605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,128,0,1,fp8,fp8,0,0.01989866668979327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,128,0,1,float16,float16,0,0.022341333329677582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,128,0,1,float16,fp8,0,0.022976001103719074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,128,0,1,fp8,fp8,0,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,128,0,1,float16,float16,0,0.516319990158081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,128,0,1,float16,fp8,0,0.5163146654764811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,128,0,1,fp8,fp8,0,0.41841065883636475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,128,0,1,float16,float16,0,0.5282453298568726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,128,0,1,float16,fp8,0,0.5224800109863281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,128,0,1,float16,float16,0,0.5627146561940511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,128,0,1,fp8,fp8,0,0.47890134652455646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,128,0,1,float16,fp8,0,0.5499573151270548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,128,0,1,float16,float16,0,0.3797866503397624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,128,0,1,fp8,fp8,0,0.5226186513900757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,128,0,1,float16,fp8,0,0.36373333136240643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,128,0,1,fp8,fp8,0,0.21974400679270426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,128,0,1,fp8,fp8,0,0.30450133482615155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,128,0,1,float16,float16,0,0.2684160073598226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,128,0,1,float16,fp8,0,0.2675093412399292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,128,0,1,float16,float16,0,0.29281065861384076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,128,0,1,float16,float16,0,0.2749920090039571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,128,0,1,float16,fp8,0,0.27211199204126996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,128,0,1,fp8,fp8,0,0.24544533093770346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,128,0,1,float16,fp8,0,0.28762133916219074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,128,0,1,float16,float16,0,0.19924799601236978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,128,0,1,fp8,fp8,0,0.2690879901250203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,128,0,1,float16,fp8,0,0.19546133279800415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,128,0,1,fp8,fp8,0,0.17108800013860068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,128,0,1,float16,float16,0,0.14471466342608133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,128,0,1,float16,fp8,0,0.14482667048772177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,128,0,1,fp8,fp8,0,0.1202880044778188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,128,0,1,float16,float16,0,0.1481226682662964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,128,0,1,float16,fp8,0,0.1461120049158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,128,0,1,fp8,fp8,0,0.12381866574287415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,128,0,1,float16,float16,0,0.15642666816711426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,128,0,1,float16,fp8,0,0.15455466508865356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,128,0,1,fp8,fp8,0,0.14004799723625183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,128,0,1,float16,float16,0,0.12085866928100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,128,0,1,float16,fp8,0,0.11587733030319214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,128,0,1,fp8,fp8,0,0.092031995455424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,128,0,1,float16,float16,0,0.07855466504891713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,128,0,1,float16,fp8,0,0.07804800073305766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,128,0,1,fp8,fp8,0,0.06841066479682922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,128,0,1,float16,float16,0,0.08349866668383281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,128,0,1,float16,fp8,0,0.08133333424727122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,128,0,1,fp8,fp8,0,0.07100800176461537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,128,0,1,float16,float16,0,0.09010666608810425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,128,0,1,float16,fp8,0,0.08846400181452434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,128,0,1,fp8,fp8,0,0.07761066655317943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,128,0,1,float16,float16,0,0.06028800209363302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,128,0,1,fp8,fp8,0,0.038575999438762665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,128,0,1,float16,fp8,0,0.05820266902446747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,128,0,1,fp8,fp8,0,0.05312533179918925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,128,0,1,float16,float16,0,0.04869333406289419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,128,0,1,float16,fp8,0,0.04916800061861674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,128,0,1,float16,float16,0,0.044826666514078774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,128,0,1,float16,fp8,0,0.04517866671085358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,128,0,1,float16,float16,0,0.047007997830708824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,128,0,1,float16,fp8,0,0.04603200157483419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,128,0,1,fp8,fp8,0,0.040991999208927155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,128,0,1,fp8,fp8,0,0.04463466505209605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,128,0,1,float16,float16,0,0.03400533397992452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,128,0,1,float16,fp8,0,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,128,0,1,fp8,fp8,0,0.03038399914900462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,128,0,1,float16,float16,0,0.030576000610987347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,128,0,1,float16,fp8,0,0.030853333572546642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,128,0,1,fp8,fp8,0,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,128,0,1,float16,float16,0,0.030853333572546642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,128,0,1,float16,fp8,0,0.030554667115211487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,128,0,1,fp8,fp8,0,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,128,0,1,float16,float16,0,0.03178666780392329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,128,0,1,float16,fp8,0,0.03183999905983607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,128,0,1,fp8,fp8,0,0.028927999238173168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,128,0,1,float16,fp8,0,0.02253866692384084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,128,0,1,float16,float16,0,0.02401600033044815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,128,0,1,float16,fp8,0,0.023999998966852825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,128,0,1,fp8,fp8,0,0.022111999491850536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,128,0,1,float16,float16,0,0.022533332308133442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,128,0,1,float16,fp8,0,0.02347733328739802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,128,0,1,fp8,fp8,0,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,128,0,1,fp8,fp8,0,0.022106667359670002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,128,0,1,float16,float16,0,0.02257599929968516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,128,0,1,float16,fp8,0,0.02013333390156428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,128,0,1,float16,fp8,0,0.02330133318901062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,128,0,1,fp8,fp8,0,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,128,0,1,float16,float16,0,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,128,0,1,float16,float16,0,0.019834666202465694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,128,0,1,fp8,fp8,0,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,128,0,1,float16,float16,0,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,128,0,1,float16,fp8,0,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,128,0,1,fp8,fp8,0,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,128,0,1,float16,float16,0,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,128,0,1,float16,fp8,0,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,128,0,1,fp8,fp8,0,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,128,0,1,float16,float16,0,0.019519999623298645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,128,0,1,float16,fp8,0,0.01985599969824155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,128,0,1,fp8,fp8,0,0.018426666657129925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,128,0,1,float16,float16,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,128,0,1,float16,fp8,0,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,128,0,1,fp8,fp8,0,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,128,0,1,fp8,fp8,0,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,128,0,1,float16,float16,0,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,128,0,1,float16,fp8,0,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,128,0,1,fp8,fp8,0,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,128,0,1,float16,float16,0,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,128,0,1,float16,fp8,0,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,128,0,1,float16,float16,0,0.01815466706951459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,128,0,1,float16,fp8,0,0.018288000176350277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,128,0,1,fp8,fp8,0,0.21655466159184775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,128,0,1,fp8,fp8,0,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,128,0,1,float16,float16,0,0.25597866376241046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,128,0,1,float16,fp8,0,0.25404266516367596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,128,0,1,float16,float16,0,0.2624746759732564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,128,0,1,float16,fp8,0,0.26226667563120526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,128,0,1,fp8,fp8,0,0.2383413314819336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,128,0,1,float16,float16,0,0.2823466658592224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,128,0,1,float16,fp8,0,0.2748960057894389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,128,0,1,fp8,fp8,0,0.26656534274419147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,128,0,1,float16,float16,0,0.2064639925956726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,128,0,1,float16,fp8,0,0.20068800449371338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,128,0,1,fp8,fp8,0,0.15873066584269205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,128,0,1,float16,float16,0,0.14114666978518167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,128,0,1,float16,fp8,0,0.1385546624660492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,128,0,1,fp8,fp8,0,0.11855467160542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,128,0,1,float16,float16,0,0.14416000247001648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,128,0,1,float16,fp8,0,0.14333333571751913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,128,0,1,fp8,fp8,0,0.12316800157229106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,128,0,1,float16,float16,0,0.15678933262825012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,128,0,1,fp8,fp8,0,0.13689600427945456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,128,0,1,float16,fp8,0,0.15339733163515726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,128,0,1,float16,float16,0,0.11250133315722148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,128,0,1,fp8,fp8,0,0.0890773336092631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,128,0,1,float16,fp8,0,0.10886399944623311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,128,0,1,float16,fp8,0,0.07439466814200084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,128,0,1,float16,float16,0,0.07398933172225952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,128,0,1,float16,float16,0,0.07791466514269511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,128,0,1,fp8,fp8,0,0.06717333197593689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,128,0,1,float16,fp8,0,0.07793599863847096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,128,0,1,float16,float16,0,0.08444799979527791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,128,0,1,fp8,fp8,0,0.07032533486684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,128,0,1,float16,fp8,0,0.08497066299120586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,128,0,1,float16,float16,0,0.059104000528653465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,128,0,1,fp8,fp8,0,0.07622399926185608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,128,0,1,float16,fp8,0,0.058090666929880776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,128,0,1,fp8,fp8,0,0.05198400219281515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,128,0,1,float16,float16,0,0.043375998735427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,128,0,1,float16,fp8,0,0.04248000184694926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,128,0,1,float16,fp8,0,0.04692266881465912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,128,0,1,fp8,fp8,0,0.038405333956082664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,128,0,1,float16,float16,0,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,128,0,1,float16,fp8,0,0.04426133135954539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,128,0,1,fp8,fp8,0,0.030954666435718536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,128,0,1,fp8,fp8,0,0.04030400017897288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,128,0,1,float16,float16,0,0.04753600060939789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,128,0,1,fp8,fp8,0,0.04436799883842468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,128,0,1,float16,float16,0,0.03339733431736628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,128,0,1,float16,fp8,0,0.03102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,128,0,1,float16,fp8,0,0.03258133431275686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,128,0,1,float16,float16,0,0.030832000076770782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,128,0,1,float16,fp8,0,0.03053866575161616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,128,0,1,fp8,fp8,0,0.026522666215896606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,128,0,1,fp8,fp8,0,0.028255999088287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,128,0,1,float16,float16,0,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,128,0,1,fp8,fp8,0,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,128,0,1,float16,float16,0,0.03200000027815501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,128,0,1,float16,fp8,0,0.03183466692765554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,128,0,1,float16,float16,0,0.02272533377011617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,128,0,1,float16,fp8,0,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,128,0,1,fp8,fp8,0,0.02189333240191142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,128,0,1,float16,float16,0,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,128,0,1,float16,fp8,0,0.02162666618824005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,128,0,1,fp8,fp8,0,0.020549333343903225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,128,0,1,float16,float16,0,0.022015998760859173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,128,0,1,float16,fp8,0,0.021888000269730885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,128,0,1,fp8,fp8,0,0.0206986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,128,0,1,float16,float16,0,0.0222080002228419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,128,0,1,float16,fp8,0,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,128,0,1,fp8,fp8,0,0.022469334304332733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,128,0,1,float16,float16,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,128,0,1,float16,fp8,0,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,128,0,1,fp8,fp8,0,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,128,0,1,float16,float16,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,128,0,1,float16,fp8,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,128,0,1,fp8,fp8,0,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,128,0,1,float16,float16,0,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,128,0,1,float16,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,128,0,1,fp8,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,128,0,1,fp8,fp8,0,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,128,0,1,float16,float16,0,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,128,0,1,float16,fp8,0,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,128,0,1,float16,float16,0,0.016704000532627106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,128,0,1,float16,fp8,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,128,0,1,fp8,fp8,0,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,128,0,1,float16,float16,0,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,128,0,1,fp8,fp8,0,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,128,0,1,float16,fp8,0,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,128,0,1,fp8,fp8,0,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,128,0,1,float16,float16,0,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,128,0,1,float16,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,128,0,1,float16,fp8,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,128,0,1,float16,float16,0,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,128,0,1,float16,fp8,0,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,128,0,1,fp8,fp8,0,0.017535999417304993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,128,0,1,float16,float16,0,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,128,0,1,fp8,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,128,0,1,float16,float16,0,0.016186666985352833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,128,0,1,float16,fp8,0,0.016362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,128,0,1,fp8,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,128,0,1,fp8,fp8,0,0.01670933390657107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,128,0,1,float16,float16,0,0.01613866661985715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,128,0,1,float16,fp8,0,0.15215999881426492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,128,0,1,float16,fp8,0,0.01634666696190834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,128,0,1,fp8,fp8,0,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,128,0,1,fp8,fp8,0,0.1625279982884725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,128,0,1,float16,float16,0,0.01591466615597407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,128,0,1,float16,float16,0,0.1653439998626709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,128,0,1,float16,fp8,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,128,0,1,float16,float16,0,0.15215466419855753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,128,0,1,float16,float16,0,0.15643733739852905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,128,0,1,float16,fp8,0,0.15686933199564615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,128,0,1,float16,fp8,0,0.10657599568367004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,128,0,1,fp8,fp8,0,0.1659999986489614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,128,0,1,float16,fp8,0,0.16172800461451212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,128,0,1,float16,fp8,0,0.08045333127180736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,128,0,1,fp8,fp8,0,0.17833600441614786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,128,0,1,float16,float16,0,0.11036266883214314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,128,0,1,fp8,fp8,0,0.10938133796056111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,128,0,1,float16,float16,0,0.07978133360544841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,128,0,1,fp8,fp8,0,0.08914666374524434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,128,0,1,float16,float16,0,0.0817440003156662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,128,0,1,float16,fp8,0,0.08186133205890656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,128,0,1,float16,fp8,0,0.05816000203291575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,128,0,1,fp8,fp8,0,0.09307199716567993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,128,0,1,float16,float16,0,0.08665600419044495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,128,0,1,float16,fp8,0,0.0869599978129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,128,0,1,fp8,fp8,0,0.09814932942390442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,128,0,1,float16,float16,0,0.04670399924119314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,128,0,1,float16,float16,0,0.05997333427270254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,128,0,1,fp8,fp8,0,0.06312533219655354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,128,0,1,float16,float16,0,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,128,0,1,float16,fp8,0,0.04643733302752177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,128,0,1,fp8,fp8,0,0.0496319979429245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,128,0,1,float16,fp8,0,0.046442667643229164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,128,0,1,float16,fp8,0,0.0316746657093366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,128,0,1,fp8,fp8,0,0.05222400029500326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,128,0,1,float16,float16,0,0.04886933167775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,128,0,1,float16,fp8,0,0.049423997600873314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,128,0,1,fp8,fp8,0,0.055311997731526695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,128,0,1,float16,float16,0,0.032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,128,0,1,float16,fp8,0,0.029461334149042766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,128,0,1,fp8,fp8,0,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,128,0,1,float16,float16,0,0.02886933336655299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,128,0,1,float16,fp8,0,0.029109333952267964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,128,0,1,fp8,fp8,0,0.031040000418821972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,128,0,1,float16,float16,0,0.029658667743206024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,128,0,1,fp8,fp8,0,0.031167998909950256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,128,0,1,float16,float16,0,0.03025600065787633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,128,0,1,float16,fp8,0,0.030405332644780476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,128,0,1,fp8,fp8,0,0.03288000077009201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,128,0,1,float16,float16,0,0.024277334411938984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,128,0,1,float16,fp8,0,0.024288001159826916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,128,0,1,fp8,fp8,0,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,128,0,1,float16,float16,0,0.023546665906906128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,128,0,1,float16,fp8,0,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,128,0,1,fp8,fp8,0,0.024154665569464367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,128,0,1,float16,fp8,0,0.02422933280467987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,128,0,1,float16,float16,0,0.023413332800070446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,128,0,1,float16,fp8,0,0.023738667368888855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,128,0,1,fp8,fp8,0,0.024160000185171764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,128,0,1,float16,float16,0,0.024143998821576435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,128,0,1,fp8,fp8,0,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,128,0,1,float16,float16,0,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,128,0,1,float16,fp8,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,128,0,1,fp8,fp8,0,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,128,0,1,float16,float16,0,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,128,0,1,float16,fp8,0,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,128,0,1,fp8,fp8,0,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,128,0,1,float16,float16,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,128,0,1,float16,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,128,0,1,fp8,fp8,0,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,128,0,1,float16,fp8,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,128,0,1,fp8,fp8,0,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,128,0,1,float16,float16,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,128,0,1,float16,fp8,0,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,128,0,1,fp8,fp8,0,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,128,0,1,float16,float16,0,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,128,0,1,float16,float16,0,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,128,0,1,float16,fp8,0,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,128,0,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,128,0,1,float16,fp8,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,128,0,1,float16,float16,0,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,128,0,1,float16,fp8,0,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,128,0,1,fp8,fp8,0,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,128,0,1,float16,float16,0,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,128,0,1,fp8,fp8,0,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,128,0,1,float16,float16,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,128,0,1,float16,float16,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,128,0,1,float16,fp8,0,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,128,0,1,fp8,fp8,0,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,128,0,1,float16,fp8,0,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,128,0,1,float16,float16,0,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,128,0,1,fp8,fp8,0,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,128,0,1,float16,float16,0,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,128,0,1,float16,fp8,0,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,128,0,1,float16,fp8,0,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,128,0,1,fp8,fp8,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,128,0,1,float16,fp8,0,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,128,0,1,fp8,fp8,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,128,0,1,float16,float16,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,128,0,1,fp8,fp8,0,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,128,0,1,float16,float16,0,0.014725333700577417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,128,0,1,float16,fp8,0,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,128,0,1,fp8,fp8,0,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,128,0,1,float16,float16,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,128,0,1,float16,fp8,0,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,128,0,1,float16,fp8,0,0.1011786659558614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,128,0,1,fp8,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,128,0,1,float16,float16,0,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,128,0,1,float16,float16,0,0.10517866412798564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,128,0,1,float16,fp8,0,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,128,0,1,fp8,fp8,0,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,128,0,1,float16,float16,0,0.10198932886123657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,128,0,1,fp8,fp8,0,0.13063466548919678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,128,0,1,float16,fp8,0,0.10454932848612468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,128,0,1,float16,fp8,0,0.07054399947325389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,128,0,1,fp8,fp8,0,0.13377599914868674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,128,0,1,float16,float16,0,0.11018666625022888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,128,0,1,float16,fp8,0,0.1088800032933553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,128,0,1,fp8,fp8,0,0.13827733198801676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,128,0,1,float16,float16,0,0.05835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,128,0,1,float16,float16,0,0.07195200026035309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,128,0,1,fp8,fp8,0,0.07483200232187907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,128,0,1,fp8,fp8,0,0.08310933411121368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,128,0,1,fp8,fp8,0,0.07780799766381581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,128,0,1,float16,float16,0,0.05715733269850413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,128,0,1,float16,float16,0,0.03731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,128,0,1,float16,fp8,0,0.03583999971548716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,128,0,1,fp8,fp8,0,0.04539200166861216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,128,0,1,float16,fp8,0,0.05653866628805796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,128,0,1,fp8,fp8,0,0.07099199791749318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,128,0,1,float16,fp8,0,0.057248001297314964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,128,0,1,float16,float16,0,0.06073066592216492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,128,0,1,float16,fp8,0,0.034186666210492454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,128,0,1,float16,fp8,0,0.05989866455396017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,128,0,1,float16,float16,0,0.03498133271932602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,128,0,1,float16,fp8,0,0.03537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,128,0,1,float16,float16,0,0.03425599883000056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,128,0,1,float16,fp8,0,0.0340639998515447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,128,0,1,fp8,fp8,0,0.04173333446184794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,128,0,1,float16,float16,0,0.03425066669782003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,128,0,1,fp8,fp8,0,0.04221333563327789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,128,0,1,fp8,fp8,0,0.043663998444875084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,128,0,1,float16,float16,0,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,128,0,1,float16,fp8,0,0.025765334566434223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,128,0,1,fp8,fp8,0,0.03014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,128,0,1,float16,float16,0,0.02362666775782903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,128,0,1,float16,fp8,0,0.023973333338896435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,128,0,1,fp8,fp8,0,0.02849599967400233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,128,0,1,float16,float16,0,0.02388266722361247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,128,0,1,float16,fp8,0,0.024325333535671234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,128,0,1,fp8,fp8,0,0.028234665592511494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,128,0,1,float16,float16,0,0.02481599897146225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,128,0,1,float16,fp8,0,0.024298667907714844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,128,0,1,fp8,fp8,0,0.029509333272775013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,128,0,1,float16,float16,0,0.01951466624935468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,128,0,1,float16,fp8,0,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,128,0,1,fp8,fp8,0,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,128,0,1,float16,float16,0,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,128,0,1,float16,fp8,0,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,128,0,1,fp8,fp8,0,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,128,0,1,float16,fp8,0,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,128,0,1,float16,float16,0,0.01817600056529045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,128,0,1,float16,fp8,0,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,128,0,1,fp8,fp8,0,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,128,0,1,float16,float16,0,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,128,0,1,float16,float16,0,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,128,0,1,fp8,fp8,0,0.016682667036851246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,128,0,1,fp8,fp8,0,0.021530665457248688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,128,0,1,float16,float16,0,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,128,0,1,fp8,fp8,0,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,128,0,1,float16,fp8,0,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,128,0,1,float16,float16,0,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,128,0,1,fp8,fp8,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,128,0,1,float16,fp8,0,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,128,0,1,float16,float16,0,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,128,0,1,float16,float16,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,128,0,1,float16,fp8,0,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,128,0,1,float16,fp8,0,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,128,0,1,fp8,fp8,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,128,0,1,float16,fp8,0,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,128,0,1,fp8,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,128,0,1,float16,float16,0,0.014309333016475042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,128,0,1,float16,fp8,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,128,0,1,float16,float16,0,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,128,0,1,fp8,fp8,0,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,128,0,1,float16,float16,0,0.01422400027513504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,128,0,1,float16,fp8,0,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,128,0,1,fp8,fp8,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,128,0,1,float16,fp8,0,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,128,0,1,fp8,fp8,0,0.016437333077192307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,128,0,1,float16,float16,0,0.014074667046467463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,128,0,1,float16,fp8,0,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,128,0,1,fp8,fp8,0,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,128,0,1,fp8,fp8,0,0.01613866661985715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,128,0,1,float16,float16,0,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,128,0,1,float16,fp8,0,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,128,0,1,fp8,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,128,0,1,float16,float16,0,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,128,0,1,float16,fp8,0,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,128,0,1,float16,float16,0,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,128,0,1,float16,fp8,0,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,128,0,1,fp8,fp8,0,0.016549333930015564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,128,0,1,float16,float16,0,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,128,0,1,float16,fp8,0,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,128,0,1,fp8,fp8,0,0.016352000335852306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,128,0,1,float16,float16,0,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,128,0,1,float16,fp8,0,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,128,0,1,fp8,fp8,0,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,128,0,1,float16,float16,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,128,0,1,float16,fp8,0,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,128,0,1,fp8,fp8,0,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,128,0,1,float16,float16,0,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,128,0,1,float16,fp8,0,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,128,0,1,fp8,fp8,0,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,128,0,1,float16,float16,0,0.08331199983755748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,128,0,1,float16,float16,0,0.08755200107892354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,128,0,1,float16,fp8,0,0.08354133367538452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,128,0,1,fp8,fp8,0,0.11340266466140747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,128,0,1,float16,float16,0,0.08506133159001668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,128,0,1,float16,fp8,0,0.08474133412043254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,128,0,1,fp8,fp8,0,0.11594133575757344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,128,0,1,float16,fp8,0,0.0862506628036499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,128,0,1,float16,float16,0,0.049914668003718056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,128,0,1,fp8,fp8,0,0.11773332953453064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,128,0,1,float16,fp8,0,0.049770668148994446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,128,0,1,float16,float16,0,0.04762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,128,0,1,fp8,fp8,0,0.06677866478761037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,128,0,1,float16,float16,0,0.04795200129350027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,128,0,1,float16,fp8,0,0.04786133269468943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,128,0,1,fp8,fp8,0,0.06284266710281372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,128,0,1,float16,fp8,0,0.047413334250450134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,128,0,1,float16,fp8,0,0.03182400017976761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,128,0,1,fp8,fp8,0,0.06285333136717479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,128,0,1,float16,float16,0,0.04868799944718679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,128,0,1,float16,fp8,0,0.04845866560935974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,128,0,1,fp8,fp8,0,0.03933866570393244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,128,0,1,fp8,fp8,0,0.06483733157316844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,128,0,1,float16,float16,0,0.03202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,128,0,1,fp8,fp8,0,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,128,0,1,fp8,fp8,0,0.040789333482583366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,128,0,1,float16,float16,0,0.030554667115211487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,128,0,1,float16,fp8,0,0.030671998858451843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,128,0,1,float16,float16,0,0.030826665461063385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,128,0,1,float16,fp8,0,0.03109866629044215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,128,0,1,fp8,fp8,0,0.02601066728432973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,128,0,1,float16,float16,0,0.03193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,128,0,1,float16,fp8,0,0.031557333966096245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,128,0,1,fp8,fp8,0,0.040720000863075256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,128,0,1,float16,float16,0,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,128,0,1,float16,fp8,0,0.021754667162895203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,128,0,1,float16,float16,0,0.020608000457286835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,128,0,1,float16,fp8,0,0.0206133338312308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,128,0,1,fp8,fp8,0,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,128,0,1,float16,float16,0,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,128,0,1,float16,fp8,0,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,128,0,1,fp8,fp8,0,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,128,0,1,float16,float16,0,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,128,0,1,float16,fp8,0,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,128,0,1,fp8,fp8,0,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,128,0,1,float16,float16,0,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,128,0,1,float16,fp8,0,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,128,0,1,fp8,fp8,0,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,128,0,1,float16,fp8,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,128,0,1,fp8,fp8,0,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,128,0,1,float16,float16,0,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,128,0,1,float16,fp8,0,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,128,0,1,fp8,fp8,0,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,128,0,1,float16,float16,0,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,128,0,1,float16,fp8,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,128,0,1,fp8,fp8,0,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,128,0,1,float16,float16,0,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,128,0,1,float16,fp8,0,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,128,0,1,fp8,fp8,0,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,128,0,1,float16,float16,0,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,128,0,1,float16,fp8,0,0.0145066666106383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,128,0,1,fp8,fp8,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,128,0,1,float16,float16,0,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,128,0,1,float16,fp8,0,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,128,0,1,fp8,fp8,0,0.016677333662907284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,128,0,1,float16,float16,0,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,128,0,1,float16,fp8,0,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,128,0,1,fp8,fp8,0,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,128,0,1,float16,float16,0,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,128,0,1,float16,fp8,0,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,128,0,1,fp8,fp8,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,128,0,1,float16,float16,0,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,128,0,1,float16,fp8,0,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,128,0,1,fp8,fp8,0,0.01624533285697301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,128,0,1,float16,float16,0,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,128,0,1,float16,fp8,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,128,0,1,fp8,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,128,0,1,float16,float16,0,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,128,0,1,float16,fp8,0,0.014490666488806406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,128,0,1,fp8,fp8,0,0.016410666207472484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,128,0,1,float16,float16,0,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,128,0,1,float16,fp8,0,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,128,0,1,fp8,fp8,0,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,128,0,1,float16,float16,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,128,0,1,float16,fp8,0,0.01422400027513504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,128,0,1,fp8,fp8,0,0.0162773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,128,0,1,fp8,fp8,0,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,128,0,1,float16,float16,0,0.01357866699496905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,128,0,1,float16,fp8,0,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,128,0,1,fp8,fp8,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,128,0,1,float16,float16,0,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,128,0,1,float16,fp8,0,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,128,0,1,float16,float16,0,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,128,0,1,float16,fp8,0,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,128,0,1,fp8,fp8,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,128,0,1,float16,float16,0,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,128,0,1,float16,fp8,0,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,128,0,1,fp8,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,128,0,1,float16,float16,0,0.013573333621025085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,128,0,1,float16,fp8,0,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,128,0,1,fp8,fp8,0,0.01597333326935768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,128,0,1,float16,float16,0,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,128,0,1,float16,fp8,0,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,128,0,1,fp8,fp8,0,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,128,0,1,fp8,fp8,0,5.444533030192058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,128,0,1,fp8,fp8,0,5.400335947672526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,128,0,1,float16,fp8,0,8.253226598103842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,128,0,1,float16,float16,0,8.625173568725586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,128,0,1,float16,float16,0,8.243146896362305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,128,0,1,float16,fp8,0,8.130191802978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,128,0,1,float16,float16,0,8.32582918802897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,128,0,1,float16,fp8,0,8.240506490071615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,128,0,1,float16,float16,0,4.367781321207683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,128,0,1,fp8,fp8,0,2.947711944580078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,128,0,1,fp8,fp8,0,5.604805628458659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,128,0,1,fp8,fp8,0,2.7843732833862305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,128,0,1,float16,fp8,0,4.254698753356934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,128,0,1,float16,float16,0,4.285034815470378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,128,0,1,float16,fp8,0,4.0370133717854815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,128,0,1,float16,float16,0,4.167050679524739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,128,0,1,fp8,fp8,0,2.8301013310750327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,128,0,1,float16,fp8,0,4.06330140431722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,128,0,1,fp8,fp8,0,1.4282719294230144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,128,0,1,float16,float16,0,2.125866731007894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,128,0,1,float16,fp8,0,2.072805404663086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,128,0,1,fp8,fp8,0,2.9030240376790366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,128,0,1,float16,float16,0,4.258362770080566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,128,0,1,float16,fp8,0,4.234031995137532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,128,0,1,fp8,fp8,0,1.3945973714192708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,128,0,1,float16,float16,0,2.150400002797445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,128,0,1,fp8,fp8,0,1.417477289835612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,128,0,1,float16,float16,0,2.1123785972595215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,128,0,1,float16,fp8,0,2.181701342264811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,128,0,1,float16,fp8,0,2.0698026021321616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,128,0,1,fp8,fp8,0,1.4525705973307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,128,0,1,float16,float16,0,1.1394826571146648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,128,0,1,float16,float16,0,2.110149383544922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,128,0,1,fp8,fp8,0,0.7073493003845215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,128,0,1,float16,fp8,0,2.1460800170898438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,128,0,1,float16,fp8,0,1.120949347813924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,128,0,1,float16,float16,0,1.0855413277943928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,128,0,1,fp8,fp8,0,0.7057440280914307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,128,0,1,float16,fp8,0,1.1104693412780762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,128,0,1,fp8,fp8,0,0.7440959612528483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,128,0,1,float16,float16,0,1.1204426288604736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,128,0,1,float16,fp8,0,1.106719970703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,128,0,1,float16,float16,0,1.1035786469777424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,128,0,1,float16,fp8,0,1.0998400052388508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,128,0,1,fp8,fp8,0,0.712165355682373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,128,0,1,fp8,fp8,0,3.2709598541259766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,128,0,1,float16,float16,0,4.907594680786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,128,0,1,fp8,fp8,0,3.1943041483561196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,128,0,1,float16,fp8,0,4.864261309305827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,128,0,1,float16,float16,0,4.950672149658203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,128,0,1,float16,fp8,0,5.137509346008301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,128,0,1,float16,float16,0,5.038271903991699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,128,0,1,fp8,fp8,0,1.6925439834594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,128,0,1,float16,float16,0,2.536832014719645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,128,0,1,float16,fp8,0,2.4329333305358887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,128,0,1,fp8,fp8,0,3.268202781677246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,128,0,1,float16,fp8,0,4.849109331766765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,128,0,1,float16,fp8,0,2.4069172541300454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,128,0,1,float16,float16,0,2.363466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,128,0,1,fp8,fp8,0,1.5938666661580403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,128,0,1,fp8,fp8,0,1.622085412343343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,128,0,1,float16,float16,0,2.3944106101989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,128,0,1,float16,float16,0,1.2812853654225667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,128,0,1,float16,fp8,0,2.5160694122314453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,128,0,1,float16,fp8,0,1.254314661026001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,128,0,1,fp8,fp8,0,1.718608061472575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,128,0,1,fp8,fp8,0,0.8735626538594564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,128,0,1,float16,float16,0,2.578549385070801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,128,0,1,float16,fp8,0,2.507882595062256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,128,0,1,float16,float16,0,1.2513386408487956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,128,0,1,fp8,fp8,0,0.8127573331197103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,128,0,1,float16,fp8,0,1.2443093458811443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,128,0,1,float16,float16,0,1.238917350769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,128,0,1,fp8,fp8,0,0.80459197362264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,128,0,1,float16,fp8,0,1.256709337234497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,128,0,1,fp8,fp8,0,0.4470133384068807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,128,0,1,float16,float16,0,0.6005493402481079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,128,0,1,float16,fp8,0,0.6053920189539591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,128,0,1,float16,float16,0,0.5820373296737671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,128,0,1,float16,float16,0,1.4232640266418457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,128,0,1,float16,fp8,0,1.2780959606170654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,128,0,1,fp8,fp8,0,0.86189866065979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,128,0,1,float16,fp8,0,0.5911680062611898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,128,0,1,fp8,fp8,0,0.42878933747609455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,128,0,1,float16,float16,0,0.582197348276774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,128,0,1,fp8,fp8,0,0.42793599764506024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,128,0,1,float16,fp8,0,0.5862720012664795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,128,0,1,fp8,fp8,0,0.43408532937367755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,128,0,1,float16,float16,0,0.580016016960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,128,0,1,float16,fp8,0,0.6234986782073975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,128,0,1,fp8,fp8,0,2.3196159998575845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,128,0,1,fp8,fp8,0,2.2288053830464682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,128,0,1,float16,float16,0,3.4918505350748696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,128,0,1,float16,fp8,0,3.559957186381022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,128,0,1,float16,float16,0,3.519136110941569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,128,0,1,float16,fp8,0,3.3459040323893228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,128,0,1,float16,float16,0,3.5779412587483725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,128,0,1,float16,fp8,0,3.5322933197021484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,128,0,1,float16,float16,0,1.7889599800109863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,128,0,1,fp8,fp8,0,1.248538653055827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,128,0,1,fp8,fp8,0,2.3402719497680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,128,0,1,float16,fp8,0,1.7659306526184082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,128,0,1,fp8,fp8,0,1.1889173189798992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,128,0,1,float16,float16,0,1.692032019297282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,128,0,1,float16,fp8,0,1.6869440078735352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,128,0,1,float16,float16,0,1.736288070678711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,128,0,1,fp8,fp8,0,1.1810399691263835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,128,0,1,float16,fp8,0,1.6949280103047688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,128,0,1,fp8,fp8,0,0.6170933246612549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,128,0,1,float16,float16,0,0.8956693013509115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,128,0,1,fp8,fp8,0,1.1895946661631267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,128,0,1,float16,fp8,0,0.9082880020141602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,128,0,1,float16,float16,0,1.683135986328125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,128,0,1,float16,fp8,0,1.6864426930745442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,128,0,1,float16,float16,0,0.8890559673309326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,128,0,1,fp8,fp8,0,0.5815840164820353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,128,0,1,float16,fp8,0,0.913920005162557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,128,0,1,fp8,fp8,0,0.5799466768900553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,128,0,1,float16,float16,0,0.8732693195343018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,128,0,1,float16,fp8,0,0.8908960024515787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,128,0,1,float16,fp8,0,0.44279468059539795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,128,0,1,float16,float16,0,0.9026827017466227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,128,0,1,fp8,fp8,0,0.588645339012146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,128,0,1,fp8,fp8,0,0.32778666416804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,128,0,1,float16,fp8,0,0.92740797996521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,128,0,1,float16,float16,0,0.43619199593861896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,128,0,1,float16,float16,0,0.4277706543604533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,128,0,1,fp8,fp8,0,0.31590400139490765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,128,0,1,float16,fp8,0,0.43077866236368817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,128,0,1,float16,float16,0,0.43853334585825604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,128,0,1,fp8,fp8,0,0.3181013266245524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,128,0,1,float16,fp8,0,0.4330933491388957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,128,0,1,float16,float16,0,0.43889065583546955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,128,0,1,float16,fp8,0,0.433077335357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,128,0,1,fp8,fp8,0,0.32016533613204956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,128,0,1,fp8,fp8,0,3.1000372568766275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,128,0,1,fp8,fp8,0,2.9614346822102866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,128,0,1,float16,float16,0,4.565930684407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,128,0,1,float16,fp8,0,4.549498558044434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,128,0,1,float16,float16,0,4.741269429524739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,128,0,1,float16,fp8,0,4.654341379801433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,128,0,1,float16,float16,0,4.611418724060059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,128,0,1,float16,float16,0,2.2925440470377603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,128,0,1,fp8,fp8,0,3.1365973154703775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,128,0,1,fp8,fp8,0,1.6217919985453289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,128,0,1,float16,fp8,0,4.587663968404134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,128,0,1,float16,fp8,0,2.2394560178120932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,128,0,1,float16,float16,0,2.170464038848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,128,0,1,float16,fp8,0,2.1561333338419595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,128,0,1,fp8,fp8,0,1.5834827423095703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,128,0,1,float16,float16,0,2.132373332977295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,128,0,1,fp8,fp8,0,1.4974826176961262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,128,0,1,float16,float16,0,1.151520013809204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,128,0,1,float16,fp8,0,2.1750826835632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,128,0,1,fp8,fp8,0,1.529626687367757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,128,0,1,float16,fp8,0,1.1542452971140544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,128,0,1,float16,float16,0,2.281925360361735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,128,0,1,fp8,fp8,0,0.832581361134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,128,0,1,float16,fp8,0,2.3273119926452637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,128,0,1,float16,float16,0,1.0929706891377766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,128,0,1,fp8,fp8,0,0.8356693585713705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,128,0,1,float16,fp8,0,1.1585546334584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,128,0,1,fp8,fp8,0,0.8270026842753092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,128,0,1,float16,float16,0,1.114853302637736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,128,0,1,float16,fp8,0,1.108021338780721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,128,0,1,float16,float16,0,1.1222240130106609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,128,0,1,fp8,fp8,0,0.7937760353088379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,128,0,1,float16,fp8,0,1.1624639828999836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,128,0,1,float16,float16,0,0.6340906620025635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,128,0,1,fp8,fp8,0,0.4230240186055501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,128,0,1,float16,fp8,0,0.6365973154703776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,128,0,1,float16,float16,0,0.6100373268127441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,128,0,1,fp8,fp8,0,0.3991359869639079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,128,0,1,float16,fp8,0,0.6128533283869425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,128,0,1,float16,float16,0,0.6127680142720541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,128,0,1,float16,fp8,0,0.6142400105794271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,128,0,1,fp8,fp8,0,0.40113600095113117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,128,0,1,float16,fp8,0,0.619759996732076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,128,0,1,float16,float16,0,0.6118933359781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,128,0,1,fp8,fp8,0,0.40590401490529376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,128,0,1,float16,float16,0,0.2927733262379964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,128,0,1,float16,fp8,0,0.2976906696955363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,128,0,1,fp8,fp8,0,0.2253920038541158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,128,0,1,float16,float16,0,0.2864266633987427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,128,0,1,float16,fp8,0,0.2888159950574239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,128,0,1,fp8,fp8,0,0.2189226746559143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,128,0,1,float16,float16,0,0.2877013285954793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,128,0,1,float16,fp8,0,0.28973867495854694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,128,0,1,fp8,fp8,0,0.2185866634051005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,128,0,1,float16,float16,0,0.29011199871699017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,128,0,1,float16,fp8,0,0.28947200377782184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,128,0,1,fp8,fp8,0,0.22217067082722983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,128,0,1,fp8,fp8,0,1.791152000427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,128,0,1,fp8,fp8,0,1.8069225947062175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,128,0,1,float16,float16,0,2.7358878453572593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,128,0,1,float16,fp8,0,2.652181307474772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,128,0,1,float16,float16,0,2.710869471232096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,128,0,1,float16,fp8,0,2.6860427856445312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,128,0,1,float16,float16,0,2.6008639335632324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,128,0,1,float16,fp8,0,2.5469600359598794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,128,0,1,float16,float16,0,1.3497546513875325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,128,0,1,fp8,fp8,0,1.8321812947591145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,128,0,1,float16,fp8,0,1.3961706161499023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,128,0,1,fp8,fp8,0,1.0216106573740642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,128,0,1,float16,float16,0,1.3014559745788574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,128,0,1,float16,fp8,0,1.2914186318715413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,128,0,1,fp8,fp8,0,0.9658293724060059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,128,0,1,float16,float16,0,1.2880160013834636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,128,0,1,fp8,fp8,0,0.9454826513926188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,128,0,1,float16,float16,0,0.7127520243326823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,128,0,1,float16,fp8,0,1.2971519629160564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,128,0,1,fp8,fp8,0,0.4984106620152791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,128,0,1,float16,fp8,0,0.7130080064137777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,128,0,1,fp8,fp8,0,0.9997653166453043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,128,0,1,float16,float16,0,1.3603146870930989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,128,0,1,float16,fp8,0,1.3126293023427327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,128,0,1,float16,float16,0,0.6775466601053873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,128,0,1,float16,float16,0,0.6673440138498942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,128,0,1,float16,fp8,0,0.6849333445231119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,128,0,1,fp8,fp8,0,0.4602773189544678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,128,0,1,float16,fp8,0,0.6750293572743734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,128,0,1,fp8,fp8,0,0.4655040105183919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,128,0,1,float16,float16,0,0.6883412996927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,128,0,1,float16,fp8,0,0.34060800075531006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,128,0,1,fp8,fp8,0,0.47464533646901447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,128,0,1,float16,float16,0,0.3397013346354167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,128,0,1,float16,fp8,0,0.6832426389058431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,128,0,1,fp8,fp8,0,0.2635626594225566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,128,0,1,float16,float16,0,0.322927991549174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,128,0,1,float16,fp8,0,0.32598400115966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,128,0,1,fp8,fp8,0,0.24682666858037314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,128,0,1,float16,float16,0,0.3237813313802083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,128,0,1,fp8,fp8,0,0.24942932526270548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,128,0,1,float16,fp8,0,0.3264533281326294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,128,0,1,float16,float16,0,0.3295946717262268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,128,0,1,fp8,fp8,0,0.2558880050977071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,128,0,1,float16,fp8,0,0.3311520020167033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,128,0,1,float16,float16,0,0.1902559995651245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,128,0,1,float16,float16,0,0.19519466161727905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,128,0,1,fp8,fp8,0,0.14480533202489218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,128,0,1,float16,float16,0,0.1907306710879008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,128,0,1,float16,fp8,0,0.1920693318049113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,128,0,1,float16,fp8,0,0.196015993754069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,128,0,1,fp8,fp8,0,0.15339733163515726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,128,0,1,float16,fp8,0,0.19006399313608804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,128,0,1,fp8,fp8,0,0.14735999703407288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,128,0,1,float16,float16,0,0.1920213301976522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,128,0,1,float16,fp8,0,0.19497066736221313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,128,0,1,fp8,fp8,0,0.1511253317197164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,128,0,1,fp8,fp8,0,1.8044853210449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,128,0,1,float16,float16,0,2.5024852752685547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,128,0,1,fp8,fp8,0,1.8196266492207844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,128,0,1,float16,fp8,0,2.488997300465902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,128,0,1,float16,float16,0,2.5468319257100425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,128,0,1,float16,fp8,0,2.520064036051432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,128,0,1,float16,float16,0,2.627781391143799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,128,0,1,float16,float16,0,1.3251413504282634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,128,0,1,float16,fp8,0,1.3318239847819011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,128,0,1,fp8,fp8,0,1.0255946318308513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,128,0,1,fp8,fp8,0,1.8794933954874675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,128,0,1,float16,float16,0,1.251482645670573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,128,0,1,float16,fp8,0,1.2644426822662354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,128,0,1,float16,fp8,0,2.5930399894714355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,128,0,1,fp8,fp8,0,0.9356479644775391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,128,0,1,fp8,fp8,0,0.9594399929046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,128,0,1,float16,float16,0,1.248149315516154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,128,0,1,float16,fp8,0,1.256709337234497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,128,0,1,float16,float16,0,0.682522694269816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,128,0,1,float16,float16,0,1.2699573040008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,128,0,1,fp8,fp8,0,0.9644532998402914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,128,0,1,float16,fp8,0,1.2768800258636475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,128,0,1,fp8,fp8,0,0.5230666796366373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,128,0,1,float16,fp8,0,0.6801706949869791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,128,0,1,float16,float16,0,0.640773336092631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,128,0,1,fp8,fp8,0,0.47786132494608563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,128,0,1,float16,fp8,0,0.6361120144526163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,128,0,1,float16,float16,0,0.6377600034077963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,128,0,1,fp8,fp8,0,0.48420798778533936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,128,0,1,float16,fp8,0,0.6386239926020304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,128,0,1,float16,float16,0,0.6489813327789307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,128,0,1,float16,float16,0,0.3704213301340739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,128,0,1,float16,fp8,0,0.6503946781158447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,128,0,1,fp8,fp8,0,0.2659093340237935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,128,0,1,fp8,fp8,0,0.49773867925008136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,128,0,1,float16,fp8,0,0.36990400155385333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,128,0,1,float16,float16,0,0.34690133730570477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,128,0,1,fp8,fp8,0,0.2426933248837789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,128,0,1,float16,fp8,0,0.348469336827596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,128,0,1,float16,float16,0,0.35075732072194415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,128,0,1,float16,fp8,0,0.35092798868815106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,128,0,1,fp8,fp8,0,0.24604799350102743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,128,0,1,float16,float16,0,0.3548746506373088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,128,0,1,float16,fp8,0,0.3574879964192708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,128,0,1,fp8,fp8,0,0.2518720030784607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,128,0,1,float16,float16,0,0.1776319940884908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,128,0,1,float16,fp8,0,0.17908799648284912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,128,0,1,fp8,fp8,0,0.14707199732462564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,128,0,1,float16,float16,0,0.1693440079689026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,128,0,1,float16,fp8,0,0.1694986621538798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,128,0,1,float16,float16,0,0.16727999846140543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,128,0,1,float16,fp8,0,0.17280532916386923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,128,0,1,float16,fp8,0,0.16742400328318277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,128,0,1,fp8,fp8,0,0.1344160040219625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,128,0,1,fp8,fp8,0,0.13693333665529886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,128,0,1,float16,float16,0,0.17157334089279175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,128,0,1,fp8,fp8,0,0.14034666617711386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,128,0,1,float16,float16,0,0.09895466764767964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,128,0,1,float16,fp8,0,0.10077866911888123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,128,0,1,float16,fp8,0,0.09757333000500996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,128,0,1,fp8,fp8,0,0.0790719985961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,128,0,1,fp8,fp8,0,0.08523199955622356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,128,0,1,float16,float16,0,0.09617066383361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,128,0,1,float16,fp8,0,0.09687999884287517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,128,0,1,fp8,fp8,0,0.077674667040507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,128,0,1,float16,float16,0,0.09601599971453349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,128,0,1,float16,float16,0,0.09754666686058044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,128,0,1,float16,fp8,0,0.09867200255393982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,128,0,1,fp8,fp8,0,0.08196799953778584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,128,0,1,fp8,fp8,0,1.1440479755401611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,128,0,1,float16,float16,0,1.5117707252502441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,128,0,1,float16,fp8,0,1.5081334114074707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,128,0,1,float16,float16,0,1.5234559377034504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,128,0,1,fp8,fp8,0,1.156549294789632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,128,0,1,float16,fp8,0,1.5217599868774414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,128,0,1,float16,float16,0,1.5768693288167317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,128,0,1,float16,fp8,0,1.5540320078531902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,128,0,1,float16,float16,0,0.8309493064880371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,128,0,1,float16,fp8,0,0.8575946489969889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,128,0,1,fp8,fp8,0,1.1792106628417969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,128,0,1,fp8,fp8,0,0.6701227029164633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,128,0,1,float16,float16,0,0.7566666603088379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,128,0,1,fp8,fp8,0,0.5876213312149048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,128,0,1,float16,fp8,0,0.7623253663380941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,128,0,1,float16,float16,0,0.7962186336517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,128,0,1,fp8,fp8,0,0.5909279982248942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,128,0,1,float16,fp8,0,0.7752799987792969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,128,0,1,float16,float16,0,0.8216213385264078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,128,0,1,fp8,fp8,0,0.6146506468454996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,128,0,1,float16,float16,0,0.4356853167215983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,128,0,1,float16,fp8,0,0.7856053511301676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,128,0,1,float16,fp8,0,0.44835734367370605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,128,0,1,float16,float16,0,0.39788798491160077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,128,0,1,fp8,fp8,0,0.3311199943224589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,128,0,1,fp8,fp8,0,0.2927146752675374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,128,0,1,float16,fp8,0,0.39824533462524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,128,0,1,float16,float16,0,0.40505067507425946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,128,0,1,fp8,fp8,0,0.2983040014902751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,128,0,1,float16,fp8,0,0.41261335213979083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,128,0,1,float16,float16,0,0.4110720157623291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,128,0,1,fp8,fp8,0,0.30874133110046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,128,0,1,float16,fp8,0,0.4121439854303996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,128,0,1,float16,float16,0,0.2170026699701945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,128,0,1,float16,fp8,0,0.21822933355967203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,128,0,1,fp8,fp8,0,0.17665600776672363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,128,0,1,float16,float16,0,0.198469340801239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,128,0,1,float16,fp8,0,0.19732266664505005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,128,0,1,fp8,fp8,0,0.1583146651585897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,128,0,1,float16,float16,0,0.19843200842539468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,128,0,1,float16,fp8,0,0.19859200716018677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,128,0,1,fp8,fp8,0,0.16115732987721762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,128,0,1,float16,float16,0,0.2032159964243571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,128,0,1,float16,fp8,0,0.20573866367340088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,128,0,1,fp8,fp8,0,0.16573333740234375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,128,0,1,float16,float16,0,0.12037332852681477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,128,0,1,float16,fp8,0,0.12102933724721272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,128,0,1,fp8,fp8,0,0.10031466682751973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,128,0,1,float16,float16,0,0.11152533690134685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,128,0,1,float16,fp8,0,0.11237333218256633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,128,0,1,fp8,fp8,0,0.08990933497746785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,128,0,1,float16,float16,0,0.11262933413187663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,128,0,1,float16,fp8,0,0.11386666695276897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,128,0,1,fp8,fp8,0,0.09217066566149394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,128,0,1,float16,float16,0,0.11487999558448792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,128,0,1,fp8,fp8,0,0.06323199967543285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,128,0,1,float16,fp8,0,0.11620799700419109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,128,0,1,fp8,fp8,0,0.09684266646703084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,128,0,1,float16,float16,0,0.07442133128643036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,128,0,1,float16,fp8,0,0.07390399773915608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,128,0,1,float16,float16,0,0.07146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,128,0,1,fp8,fp8,0,0.05874133110046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,128,0,1,float16,fp8,0,0.07270933190981548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,128,0,1,fp8,fp8,0,0.05816533168156942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,128,0,1,float16,float16,0,0.0718506673971812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,128,0,1,float16,fp8,0,0.07283199826876323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,128,0,1,float16,float16,0,0.07191466788450877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,128,0,1,fp8,fp8,0,0.06006933252016703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,128,0,1,float16,fp8,0,0.07328533132870992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,128,0,1,fp8,fp8,0,1.260314702987671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,128,0,1,float16,float16,0,1.5906880696614583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,128,0,1,float16,fp8,0,1.5750667254130046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,128,0,1,fp8,fp8,0,1.2762239774068196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,128,0,1,float16,float16,0,1.5986240704854329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,128,0,1,float16,fp8,0,1.5991840362548828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,128,0,1,float16,float16,0,1.6536266009012859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,128,0,1,float16,fp8,0,1.64081605275472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,128,0,1,float16,float16,0,0.890671968460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,128,0,1,fp8,fp8,0,0.712287982304891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,128,0,1,float16,fp8,0,0.8754506905873617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,128,0,1,float16,float16,0,0.786730686823527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,128,0,1,fp8,fp8,0,1.309381326039632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,128,0,1,float16,fp8,0,0.7984000047047933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,128,0,1,fp8,fp8,0,0.6522239844004313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,128,0,1,float16,float16,0,0.8100693225860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,128,0,1,fp8,fp8,0,0.6451626618703207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,128,0,1,float16,fp8,0,0.8072160085042318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,128,0,1,float16,float16,0,0.4549119869867961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,128,0,1,float16,float16,0,0.8293333053588867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,128,0,1,fp8,fp8,0,0.6651839812596639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,128,0,1,float16,fp8,0,0.455728014310201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,128,0,1,float16,fp8,0,0.8239093621571859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,128,0,1,fp8,fp8,0,0.32814933856328327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,128,0,1,fp8,fp8,0,0.3702826499938965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,128,0,1,float16,float16,0,0.401093324025472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,128,0,1,float16,fp8,0,0.4184639851252238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,128,0,1,float16,float16,0,0.40571733315785724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,128,0,1,float16,fp8,0,0.4089173475901286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,128,0,1,fp8,fp8,0,0.33585067590077716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,128,0,1,float16,float16,0,0.4231040080388387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,128,0,1,float16,fp8,0,0.4195573329925537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,128,0,1,float16,float16,0,0.24390933911005655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,128,0,1,fp8,fp8,0,0.3454506794611613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,128,0,1,float16,fp8,0,0.24503999948501587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,128,0,1,fp8,fp8,0,0.19129067659378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,128,0,1,float16,float16,0,0.21864533424377441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,128,0,1,float16,fp8,0,0.2208906610806783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,128,0,1,float16,fp8,0,0.22033067544301352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,128,0,1,fp8,fp8,0,0.16637866695721945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,128,0,1,float16,float16,0,0.21951999266942343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,128,0,1,float16,fp8,0,0.1229759951432546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,128,0,1,fp8,fp8,0,0.1689866582552592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,128,0,1,float16,float16,0,0.2281386653582255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,128,0,1,float16,fp8,0,0.22757333517074585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,128,0,1,fp8,fp8,0,0.17573332786560059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,128,0,1,float16,float16,0,0.12187199791272481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,128,0,1,fp8,fp8,0,0.1058079997698466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,128,0,1,float16,float16,0,0.10871466994285583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,128,0,1,float16,fp8,0,0.10889066259066264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,128,0,1,fp8,fp8,0,0.09269332885742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,128,0,1,float16,fp8,0,0.11460266510645549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,128,0,1,float16,float16,0,0.10972799857457478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,128,0,1,fp8,fp8,0,0.09880533814430237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,128,0,1,float16,fp8,0,0.11044266819953918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,128,0,1,fp8,fp8,0,0.0950879951318105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,128,0,1,float16,float16,0,0.11268267035484314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,128,0,1,float16,float16,0,0.0666720022757848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,128,0,1,float16,fp8,0,0.06821866830190022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,128,0,1,fp8,fp8,0,0.0618453323841095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,128,0,1,float16,fp8,0,0.06321066617965698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,128,0,1,float16,float16,0,0.062234664956728615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,128,0,1,float16,fp8,0,0.06227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,128,0,1,fp8,fp8,0,0.05300800005594889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,128,0,1,float16,float16,0,0.053445334235827126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,128,0,1,float16,float16,0,0.06222933530807495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,128,0,1,fp8,fp8,0,0.05461333195368449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,128,0,1,float16,float16,0,0.06288533409436543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,128,0,1,float16,fp8,0,0.06486933430035909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,128,0,1,fp8,fp8,0,0.05693333347638448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,128,0,1,float16,fp8,0,0.05367999772230784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,128,0,1,fp8,fp8,0,0.04520533482233683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,128,0,1,float16,float16,0,0.0524586687485377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,128,0,1,fp8,fp8,0,0.04404266675313314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,128,0,1,float16,fp8,0,0.05268266797065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,128,0,1,fp8,fp8,0,0.043765331308046974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,128,0,1,float16,float16,0,0.05269333223501841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,128,0,1,float16,fp8,0,0.05439466734727224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,128,0,1,float16,float16,0,0.05256533126036326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,128,0,1,float16,fp8,0,0.05313600103060404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,128,0,1,fp8,fp8,0,0.044122666120529175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,128,0,1,float16,float16,0,0.9930133024851481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,128,0,1,float16,fp8,0,0.9789919853210449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,128,0,1,fp8,fp8,0,0.8314826488494873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,128,0,1,float16,fp8,0,1.008069356282552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,128,0,1,float16,float16,0,1.003226677576701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,128,0,1,fp8,fp8,0,0.8406666914621989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,128,0,1,float16,float16,0,1.0339199701944988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,128,0,1,float16,float16,0,0.5730773210525513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,128,0,1,float16,fp8,0,1.018506685892741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,128,0,1,fp8,fp8,0,0.871999979019165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,128,0,1,float16,fp8,0,0.5728746652603149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,128,0,1,fp8,fp8,0,0.4936586618423462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,128,0,1,float16,float16,0,0.4983146588007609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,128,0,1,float16,float16,0,0.5104586680730184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,128,0,1,float16,fp8,0,0.49982933203379315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,128,0,1,fp8,fp8,0,0.42823465665181476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,128,0,1,float16,fp8,0,0.5047253370285034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,128,0,1,fp8,fp8,0,0.43321601549784344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,128,0,1,float16,float16,0,0.5236426591873169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,128,0,1,float16,fp8,0,0.5267359813054403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,128,0,1,float16,float16,0,0.30080000559488934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,128,0,1,fp8,fp8,0,0.44172799587249756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,128,0,1,float16,fp8,0,0.2973546584447225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,128,0,1,float16,float16,0,0.2677066723505656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,128,0,1,float16,fp8,0,0.26658666133880615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,128,0,1,fp8,fp8,0,0.26765332619349164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,128,0,1,float16,float16,0,0.261957327524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,128,0,1,float16,fp8,0,0.26174400250116986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,128,0,1,fp8,fp8,0,0.2097813288370768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,128,0,1,fp8,fp8,0,0.21448000272115073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,128,0,1,float16,float16,0,0.27270932992299396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,128,0,1,float16,fp8,0,0.27692800760269165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,128,0,1,fp8,fp8,0,0.22510399421056113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,128,0,1,fp8,fp8,0,0.11515733599662781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,128,0,1,float16,float16,0,0.1623306671778361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,128,0,1,float16,fp8,0,0.16336533427238464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,128,0,1,fp8,fp8,0,0.13333866993586221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,128,0,1,float16,float16,0,0.13297067085901895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,128,0,1,float16,fp8,0,0.13329066832860312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,128,0,1,float16,float16,0,0.1370186706384023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,128,0,1,float16,fp8,0,0.13762666781743368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,128,0,1,fp8,fp8,0,0.1188213328520457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,128,0,1,float16,float16,0,0.1406880021095276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,128,0,1,float16,float16,0,0.07293866574764252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,128,0,1,float16,fp8,0,0.14285866419474283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,128,0,1,fp8,fp8,0,0.1220266620318095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,128,0,1,float16,float16,0,0.07647466659545898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,128,0,1,float16,float16,0,0.08528000116348267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,128,0,1,float16,fp8,0,0.08450667063395183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,128,0,1,fp8,fp8,0,0.07585600018501282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,128,0,1,float16,fp8,0,0.07398400207360585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,128,0,1,fp8,fp8,0,0.06344000001748402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,128,0,1,float16,fp8,0,0.07564799984296162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,128,0,1,fp8,fp8,0,0.06758399804433186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,128,0,1,float16,float16,0,0.07801066835721333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,128,0,1,float16,float16,0,0.045653333266576133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,128,0,1,float16,fp8,0,0.07911466558774312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,128,0,1,fp8,fp8,0,0.07086400190989177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,128,0,1,float16,fp8,0,0.047168001532554626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,128,0,1,float16,float16,0,0.04868266483147939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,128,0,1,fp8,fp8,0,0.04190400242805481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,128,0,1,float16,fp8,0,0.04914666712284088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,128,0,1,fp8,fp8,0,0.04488533238569895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,128,0,1,float16,fp8,0,0.04643733302752177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,128,0,1,fp8,fp8,0,0.03974399964014689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,128,0,1,float16,float16,0,0.04570133487383524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,128,0,1,float16,float16,0,0.04660266637802124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,128,0,1,float16,fp8,0,0.04819199939568838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,128,0,1,fp8,fp8,0,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,128,0,1,float16,float16,0,0.04238399863243103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,128,0,1,float16,fp8,0,0.04326933125654856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,128,0,1,fp8,fp8,0,0.03735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,128,0,1,float16,float16,0,0.04223466912905375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,128,0,1,float16,fp8,0,0.04293866455554962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,128,0,1,fp8,fp8,0,0.036618667344252266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,128,0,1,float16,float16,0,0.04286933441956838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,128,0,1,float16,fp8,0,0.04236799975236257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,128,0,1,fp8,fp8,0,0.036229332288106285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,128,0,1,float16,float16,0,1.0125813484191895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,128,0,1,float16,fp8,0,1.0111253261566162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,128,0,1,fp8,fp8,0,0.8773866494496664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,128,0,1,float16,fp8,0,1.0140906969706218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,128,0,1,float16,float16,0,0.042912001411120095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,128,0,1,float16,fp8,0,0.043247997760772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,128,0,1,fp8,fp8,0,0.03658133248488108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,128,0,1,float16,float16,0,1.0285653273264568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,128,0,1,fp8,fp8,0,0.8983413378397623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,128,0,1,fp8,fp8,0,1.0140106678009033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,128,0,1,float16,float16,0,1.094218651453654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,128,0,1,float16,fp8,0,1.071679989496867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,128,0,1,float16,float16,0,0.6401120026906332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,128,0,1,float16,fp8,0,0.6198293368021647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,128,0,1,fp8,fp8,0,0.5553013483683268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,128,0,1,float16,float16,0,0.5241653521855673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,128,0,1,float16,float16,0,0.516485333442688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,128,0,1,float16,fp8,0,0.5138346751530966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,128,0,1,fp8,fp8,0,0.4480373462041219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,128,0,1,float16,fp8,0,0.5185866753260294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,128,0,1,fp8,fp8,0,0.45305601755777997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,128,0,1,float16,float16,0,0.3349013328552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,128,0,1,float16,float16,0,0.562447985013326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,128,0,1,float16,fp8,0,0.5548693339029948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,128,0,1,fp8,fp8,0,0.5052693287531534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,128,0,1,float16,fp8,0,0.3233333428700765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,128,0,1,fp8,fp8,0,0.2821279962857564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,128,0,1,float16,float16,0,0.27029865980148315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,128,0,1,float16,fp8,0,0.2695840001106262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,128,0,1,float16,float16,0,0.2765226761500041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,128,0,1,fp8,fp8,0,0.23322665691375732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,128,0,1,float16,fp8,0,0.1750826636950175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,128,0,1,float16,fp8,0,0.2759840091069539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,128,0,1,fp8,fp8,0,0.23449599742889404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,128,0,1,float16,float16,0,0.29286932945251465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,128,0,1,float16,fp8,0,0.28995732466379803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,128,0,1,fp8,fp8,0,0.1164533297220866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,128,0,1,fp8,fp8,0,0.2606666684150696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,128,0,1,float16,fp8,0,0.15090133746465048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,128,0,1,float16,float16,0,0.18201599518458048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,128,0,1,fp8,fp8,0,0.1455359955628713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,128,0,1,float16,float16,0,0.1481013298034668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,128,0,1,float16,fp8,0,0.14845333496729532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,128,0,1,float16,float16,0,0.14994666973749796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,128,0,1,fp8,fp8,0,0.11909866333007812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,128,0,1,float16,float16,0,0.16029333074887595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,128,0,1,float16,fp8,0,0.15749333302179971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,128,0,1,fp8,fp8,0,0.1267626682917277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,128,0,1,float16,float16,0,0.09548266728719075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,128,0,1,float16,fp8,0,0.09278933207194011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,128,0,1,fp8,fp8,0,0.08085866769154866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,128,0,1,float16,float16,0,0.07656000057856242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,128,0,1,float16,fp8,0,0.0768746683994929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,128,0,1,float16,fp8,0,0.08111999928951263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,128,0,1,fp8,fp8,0,0.06479466458161671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,128,0,1,float16,float16,0,0.07830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,128,0,1,float16,fp8,0,0.07744533320267995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,128,0,1,fp8,fp8,0,0.06718400120735168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,128,0,1,float16,float16,0,0.08196799953778584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,128,0,1,fp8,fp8,0,0.07111999889214833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,128,0,1,float16,float16,0,0.05020800232887268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,128,0,1,float16,fp8,0,0.04876266419887543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,128,0,1,fp8,fp8,0,0.045082668463389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,128,0,1,float16,float16,0,0.042992000778516136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,128,0,1,float16,fp8,0,0.04317333300908407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,128,0,1,fp8,fp8,0,0.03665599972009659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,128,0,1,float16,float16,0,0.0436106671889623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,128,0,1,float16,fp8,0,0.04394133388996124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,128,0,1,fp8,fp8,0,0.037578667203585304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,128,0,1,float16,float16,0,0.046207999189694725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,128,0,1,float16,fp8,0,0.045082668463389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,128,0,1,fp8,fp8,0,0.04085333396991094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,128,0,1,float16,float16,0,0.034245334565639496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,128,0,1,float16,fp8,0,0.03348266581694285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,128,0,1,fp8,fp8,0,0.02940266579389572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,128,0,1,float16,float16,0,0.032373333970705666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,128,0,1,float16,fp8,0,0.032058666149775185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,128,0,1,fp8,fp8,0,0.0277813325325648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,128,0,1,float16,float16,0,0.0330079992612203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,128,0,1,float16,fp8,0,0.03293866664171219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,128,0,1,fp8,fp8,0,0.02792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,128,0,1,float16,float16,0,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,128,0,1,float16,fp8,0,0.03388266762097677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,128,0,1,fp8,fp8,0,0.02888533224662145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,128,0,1,float16,float16,0,0.030095999439557392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,128,0,1,float16,fp8,0,0.030773334205150604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,128,0,1,fp8,fp8,0,0.025850666066010792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,128,0,1,float16,float16,0,0.029215998947620392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,128,0,1,float16,fp8,0,0.029557332396507263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,128,0,1,float16,fp8,0,0.029103999336560566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,128,0,1,fp8,fp8,0,0.02499199906984965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,128,0,1,float16,float16,0,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,128,0,1,fp8,fp8,0,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,128,0,1,float16,float16,0,0.030117332935333252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,128,0,1,float16,fp8,0,0.029989334444204967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,128,0,1,fp8,fp8,0,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,128,0,1,float16,float16,0,0.794426679611206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,128,0,1,float16,fp8,0,0.7902239958445231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,128,0,1,fp8,fp8,0,0.7124586900075277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,128,0,1,float16,float16,0,0.8184426625569662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,128,0,1,float16,fp8,0,0.8136586348215739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,128,0,1,fp8,fp8,0,0.7307999928792318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,128,0,1,float16,float16,0,0.8840160369873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,128,0,1,float16,float16,0,0.5380533138910929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,128,0,1,float16,fp8,0,0.8613920211791992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,128,0,1,float16,fp8,0,0.40751465161641437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,128,0,1,float16,fp8,0,0.5200639963150024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,128,0,1,fp8,fp8,0,0.841200033823649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,128,0,1,fp8,fp8,0,0.472981333732605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,128,0,1,float16,float16,0,0.40724798043568927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,128,0,1,fp8,fp8,0,0.3616960048675537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,128,0,1,float16,float16,0,0.417520006497701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,128,0,1,float16,fp8,0,0.4150346517562866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,128,0,1,float16,fp8,0,0.4451253414154053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,128,0,1,fp8,fp8,0,0.3677599827448527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,128,0,1,float16,float16,0,0.4546240170796712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,128,0,1,fp8,fp8,0,0.4196213483810425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,128,0,1,float16,float16,0,0.27989866336186725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,128,0,1,float16,fp8,0,0.2706186572710673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,128,0,1,float16,float16,0,0.21372799078623453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,128,0,1,fp8,fp8,0,0.23937066396077475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,128,0,1,float16,fp8,0,0.21607466538747153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,128,0,1,fp8,fp8,0,0.190394659837087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,128,0,1,float16,float16,0,0.22036266326904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,128,0,1,float16,fp8,0,0.2179786761601766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,128,0,1,fp8,fp8,0,0.1930826703707377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,128,0,1,float16,float16,0,0.23880533377329508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,128,0,1,float16,fp8,0,0.23457066218058267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,128,0,1,fp8,fp8,0,0.2135093410809835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,128,0,1,float16,float16,0,0.1509866714477539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,128,0,1,float16,fp8,0,0.14629866679509482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,128,0,1,fp8,fp8,0,0.12723199526468912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,128,0,1,float16,float16,0,0.11743467052777608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,128,0,1,float16,fp8,0,0.11754666765530904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,128,0,1,fp8,fp8,0,0.097680002450943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,128,0,1,float16,float16,0,0.11968533198038737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,128,0,1,float16,fp8,0,0.12049600481987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,128,0,1,fp8,fp8,0,0.09876799583435059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,128,0,1,float16,float16,0,0.13145599762598673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,128,0,1,float16,fp8,0,0.12731200456619263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,128,0,1,fp8,fp8,0,0.10566400488217671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,128,0,1,float16,float16,0,0.08494399984677632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,128,0,1,float16,fp8,0,0.08170133332411449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,128,0,1,fp8,fp8,0,0.07125866909821828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,128,0,1,float16,float16,0,0.06261866788069408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,128,0,1,float16,fp8,0,0.06251733501752217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,128,0,1,float16,fp8,0,0.06858666737874348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,128,0,1,fp8,fp8,0,0.055402666330337524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,128,0,1,float16,float16,0,0.06479999919732411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,128,0,1,float16,fp8,0,0.06404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,128,0,1,fp8,fp8,0,0.05701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,128,0,1,float16,float16,0,0.06852266689141591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,128,0,1,fp8,fp8,0,0.06116800010204315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,128,0,1,float16,float16,0,0.043968002001444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,128,0,1,float16,float16,0,0.03826133410135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,128,0,1,float16,fp8,0,0.03763733307520548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,128,0,1,fp8,fp8,0,0.0323840007185936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,128,0,1,float16,fp8,0,0.04267199834187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,128,0,1,fp8,fp8,0,0.04161600023508072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,128,0,1,float16,float16,0,0.03665599972009659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,128,0,1,float16,fp8,0,0.03654933224121729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,128,0,1,fp8,fp8,0,0.031744000812371574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,128,0,1,float16,float16,0,0.04018666595220566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,128,0,1,float16,fp8,0,0.039461334546407066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,128,0,1,fp8,fp8,0,0.03506666670242945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,128,0,1,float16,float16,0,0.027808000644048054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,128,0,1,float16,fp8,0,0.028021333118279774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,128,0,1,fp8,fp8,0,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,128,0,1,fp8,fp8,0,0.024773334463437397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,128,0,1,float16,float16,0,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,128,0,1,float16,fp8,0,0.026122666895389557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,128,0,1,fp8,fp8,0,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,128,0,1,float16,float16,0,0.02651199946800868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,128,0,1,float16,fp8,0,0.025994665920734406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,128,0,1,float16,float16,0,0.02757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,128,0,1,float16,fp8,0,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,128,0,1,fp8,fp8,0,0.024080000817775726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,128,0,1,fp8,fp8,0,0.020245333512624104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,128,0,1,float16,float16,0,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,128,0,1,float16,fp8,0,0.02385599911212921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,128,0,1,fp8,fp8,0,0.021717332303524017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,128,0,1,float16,float16,0,0.02276800076166789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,128,0,1,float16,fp8,0,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,128,0,1,float16,float16,0,0.022245332598686218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,128,0,1,float16,fp8,0,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,128,0,1,fp8,fp8,0,0.02048533285657565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,128,0,1,float16,float16,0,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,128,0,1,float16,fp8,0,0.023397333920001984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,128,0,1,fp8,fp8,0,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,128,0,1,float16,float16,0,0.021802666286627453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,128,0,1,float16,float16,0,0.020746666938066483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,128,0,1,float16,fp8,0,0.021717332303524017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,128,0,1,fp8,fp8,0,0.019621333728233974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,128,0,1,fp8,fp8,0,0.019760000209013622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,128,0,1,float16,float16,0,0.02176533391078313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,128,0,1,float16,fp8,0,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,128,0,1,fp8,fp8,0,0.019674666225910187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,128,0,1,float16,fp8,0,0.022240000466505688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,128,0,1,fp8,fp8,0,0.29445334275563556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,128,0,1,float16,float16,0,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,128,0,1,float16,fp8,0,0.022005334496498108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,128,0,1,fp8,fp8,0,0.019909333437681198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,128,0,1,float16,float16,0,0.3556053241093953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,128,0,1,float16,fp8,0,0.35426131884257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,128,0,1,float16,float16,0,0.365882674853007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,128,0,1,float16,fp8,0,0.3635573387145996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,128,0,1,fp8,fp8,0,0.3022613326708476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,128,0,1,float16,float16,0,0.40269867579142254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,128,0,1,float16,fp8,0,0.3940800031026204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,128,0,1,float16,float16,0,0.26075732707977295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,128,0,1,fp8,fp8,0,0.360154668490092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,128,0,1,float16,fp8,0,0.25005332628885907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,128,0,1,fp8,fp8,0,0.2119040091832479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,128,0,1,float16,float16,0,0.18682666619618735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,128,0,1,fp8,fp8,0,0.16250666975975037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,128,0,1,fp8,fp8,0,0.15868266423543295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,128,0,1,float16,fp8,0,0.20805867513020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,128,0,1,float16,float16,0,0.14167466759681702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,128,0,1,fp8,fp8,0,0.17938133080800375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,128,0,1,float16,fp8,0,0.185914675394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,128,0,1,float16,float16,0,0.19460266828536987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,128,0,1,float16,fp8,0,0.19236799081166586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,128,0,1,float16,float16,0,0.2136426568031311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,128,0,1,float16,fp8,0,0.13784000277519226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,128,0,1,fp8,fp8,0,0.11931733290354411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,128,0,1,float16,float16,0,0.10427199800809224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,128,0,1,float16,fp8,0,0.10389332969983418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,128,0,1,fp8,fp8,0,0.0888159970442454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,128,0,1,fp8,fp8,0,0.0977280040582021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,128,0,1,float16,float16,0,0.10831466317176819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,128,0,1,float16,fp8,0,0.10612266262372334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,128,0,1,fp8,fp8,0,0.08985599875450134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,128,0,1,float16,float16,0,0.11756267150243123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,128,0,1,float16,fp8,0,0.11544533570607503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,128,0,1,float16,float16,0,0.0825439989566803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,128,0,1,float16,float16,0,0.05796800057093302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,128,0,1,float16,fp8,0,0.0791733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,128,0,1,fp8,fp8,0,0.0661599983771642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,128,0,1,float16,float16,0,0.05726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,128,0,1,float16,fp8,0,0.05694933235645294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,128,0,1,fp8,fp8,0,0.0566240002711614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,128,0,1,fp8,fp8,0,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,128,0,1,float16,fp8,0,0.058703998724619545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,128,0,1,fp8,fp8,0,0.0521066685517629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,128,0,1,float16,float16,0,0.06421866516272227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,128,0,1,float16,fp8,0,0.06307733555634816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,128,0,1,float16,float16,0,0.04038933416207632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,128,0,1,float16,fp8,0,0.038704000413417816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,128,0,1,float16,fp8,0,0.03382399926582972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,128,0,1,fp8,fp8,0,0.030645333230495453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,128,0,1,fp8,fp8,0,0.03904533386230469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,128,0,1,float16,fp8,0,0.03618133316437403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,128,0,1,fp8,fp8,0,0.0335359995563825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,128,0,1,float16,float16,0,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,128,0,1,float16,fp8,0,0.02604266752799352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,128,0,1,float16,float16,0,0.033530667424201965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,128,0,1,float16,fp8,0,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,128,0,1,fp8,fp8,0,0.029829333225886028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,128,0,1,float16,float16,0,0.034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,128,0,1,float16,float16,0,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,128,0,1,float16,fp8,0,0.023973333338896435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,128,0,1,float16,float16,0,0.03682666768630346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,128,0,1,fp8,fp8,0,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,128,0,1,float16,float16,0,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,128,0,1,float16,fp8,0,0.02383466561635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,128,0,1,fp8,fp8,0,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,128,0,1,fp8,fp8,0,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,128,0,1,fp8,fp8,0,0.019402666638294857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,128,0,1,float16,float16,0,0.019839999576409657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,128,0,1,float16,fp8,0,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,128,0,1,fp8,fp8,0,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,128,0,1,float16,float16,0,0.024906667570273083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,128,0,1,float16,fp8,0,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,128,0,1,fp8,fp8,0,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,128,0,1,float16,float16,0,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,128,0,1,float16,fp8,0,0.020693333198626835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,128,0,1,fp8,fp8,0,0.018810667097568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,128,0,1,float16,float16,0,0.019813333948453266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,128,0,1,float16,fp8,0,0.019802667200565338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,128,0,1,fp8,fp8,0,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,128,0,1,float16,float16,0,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,128,0,1,float16,fp8,0,0.020202666521072388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,128,0,1,float16,float16,0,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,128,0,1,float16,fp8,0,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,128,0,1,fp8,fp8,0,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,128,0,1,float16,float16,0,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,128,0,1,float16,float16,0,0.01859733338157336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,128,0,1,float16,fp8,0,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,128,0,1,fp8,fp8,0,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,128,0,1,float16,float16,0,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,128,0,1,float16,fp8,0,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,128,0,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,128,0,1,float16,fp8,0,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,128,0,1,fp8,fp8,0,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,128,0,1,float16,float16,0,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,128,0,1,float16,fp8,0,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,128,0,1,fp8,fp8,0,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,128,0,1,float16,float16,0,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,128,0,1,float16,fp8,0,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,128,0,1,fp8,fp8,0,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,128,0,1,float16,float16,0,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,128,0,1,float16,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,128,0,1,fp8,fp8,0,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,128,0,1,float16,float16,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,128,0,1,float16,fp8,0,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,128,0,1,fp8,fp8,0,0.01670933390657107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,128,0,1,float16,fp8,0,0.17740267515182495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,128,0,1,float16,float16,0,0.17774933576583862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,128,0,1,fp8,fp8,0,0.15571733315785727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,128,0,1,fp8,fp8,0,0.1818880041440328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,128,0,1,float16,fp8,0,0.20657066504160562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,128,0,1,float16,float16,0,0.18702934185663858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,128,0,1,float16,fp8,0,0.18451199928919473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,128,0,1,float16,float16,0,0.10145599643389384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,128,0,1,fp8,fp8,0,0.16146666804949442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,128,0,1,float16,float16,0,0.2098026672999064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,128,0,1,float16,float16,0,0.14527466893196106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,128,0,1,fp8,fp8,0,0.11059733231862386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,128,0,1,float16,fp8,0,0.14055466651916504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,128,0,1,float16,fp8,0,0.1018346647421519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,128,0,1,fp8,fp8,0,0.08724799752235413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,128,0,1,float16,float16,0,0.10717866818110149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,128,0,1,float16,fp8,0,0.10408000151316325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,128,0,1,fp8,fp8,0,0.09027733405431111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,128,0,1,float16,float16,0,0.11947199702262878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,128,0,1,float16,float16,0,0.05413866539796194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,128,0,1,float16,fp8,0,0.1158026655515035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,128,0,1,fp8,fp8,0,0.09798933068911235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,128,0,1,float16,float16,0,0.07646400233109792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,128,0,1,float16,fp8,0,0.07445333401362102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,128,0,1,fp8,fp8,0,0.052527998884518944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,128,0,1,fp8,fp8,0,0.064410666624705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,128,0,1,float16,fp8,0,0.053930665055910744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,128,0,1,fp8,fp8,0,0.04958933095137278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,128,0,1,float16,float16,0,0.05650666852792104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,128,0,1,float16,fp8,0,0.055946667989095054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,128,0,1,float16,float16,0,0.062277331948280334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,128,0,1,float16,fp8,0,0.06246933341026306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,128,0,1,fp8,fp8,0,0.057349334160486855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,128,0,1,float16,float16,0,0.039162665605545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,128,0,1,float16,fp8,0,0.03899200012286504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,128,0,1,float16,fp8,0,0.03384533276160558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,128,0,1,fp8,fp8,0,0.0377813329299291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,128,0,1,float16,float16,0,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,128,0,1,float16,fp8,0,0.033370666205883026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,128,0,1,fp8,fp8,0,0.03002133220434189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,128,0,1,float16,float16,0,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,128,0,1,fp8,fp8,0,0.03046400099992752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,128,0,1,float16,float16,0,0.035216001172860466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,128,0,1,float16,fp8,0,0.035114665826161705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,128,0,1,fp8,fp8,0,0.03277866790692011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,128,0,1,float16,float16,0,0.024879999458789825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,128,0,1,float16,fp8,0,0.024165332317352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,128,0,1,float16,fp8,0,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,128,0,1,fp8,fp8,0,0.022602667411168415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,128,0,1,float16,float16,0,0.022384000321229298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,128,0,1,float16,fp8,0,0.022800001005331676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,128,0,1,fp8,fp8,0,0.02086399992307027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,128,0,1,float16,float16,0,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,128,0,1,fp8,fp8,0,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,128,0,1,float16,float16,0,0.02378133436044057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,128,0,1,float16,fp8,0,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,128,0,1,fp8,fp8,0,0.022309333086013794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,128,0,1,float16,float16,0,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,128,0,1,float16,fp8,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,128,0,1,float16,fp8,0,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,128,0,1,fp8,fp8,0,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,128,0,1,float16,float16,0,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,128,0,1,float16,fp8,0,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,128,0,1,fp8,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,128,0,1,float16,float16,0,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,128,0,1,fp8,fp8,0,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,128,0,1,float16,fp8,0,0.018645333747069042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,128,0,1,fp8,fp8,0,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,128,0,1,float16,float16,0,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,128,0,1,float16,fp8,0,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,128,0,1,fp8,fp8,0,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,128,0,1,float16,float16,0,0.016410666207472484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,128,0,1,float16,fp8,0,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,128,0,1,fp8,fp8,0,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,128,0,1,float16,float16,0,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,128,0,1,float16,fp8,0,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,128,0,1,fp8,fp8,0,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,128,0,1,float16,float16,0,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,128,0,1,float16,fp8,0,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,128,0,1,fp8,fp8,0,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,128,0,1,float16,fp8,0,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,128,0,1,float16,float16,0,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,128,0,1,float16,fp8,0,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,128,0,1,fp8,fp8,0,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,128,0,1,fp8,fp8,0,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,128,0,1,float16,float16,0,0.01594666639963786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,128,0,1,float16,fp8,0,0.016330666840076447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,128,0,1,fp8,fp8,0,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,128,0,1,float16,float16,0,0.015909332782030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,128,0,1,fp8,fp8,0,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,128,0,1,float16,float16,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,128,0,1,float16,fp8,0,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,128,0,1,float16,float16,0,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,128,0,1,float16,fp8,0,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,128,0,1,fp8,fp8,0,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,128,0,1,fp8,fp8,0,0.01653333380818367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,128,0,1,float16,float16,0,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,128,0,1,float16,fp8,0,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,128,0,1,fp8,fp8,0,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,128,0,1,float16,float16,0,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,128,0,1,float16,float16,0,0.1079200009504954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,128,0,1,float16,fp8,0,0.01616000011563301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,128,0,1,float16,float16,0,0.015568000574906668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,128,0,1,float16,fp8,0,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,128,0,1,fp8,fp8,0,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,128,0,1,float16,fp8,0,0.10716799894968669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,128,0,1,fp8,fp8,0,0.11614400148391724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,128,0,1,float16,fp8,0,0.1116426686445872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,128,0,1,float16,float16,0,0.11261333028475444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,128,0,1,fp8,fp8,0,0.11820800105730693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,128,0,1,float16,float16,0,0.12059733271598816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,128,0,1,float16,fp8,0,0.12009066343307495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,128,0,1,fp8,fp8,0,0.12610133488972983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,128,0,1,float16,float16,0,0.07704533139864604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,128,0,1,float16,fp8,0,0.07419733206431071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,128,0,1,fp8,fp8,0,0.07815999786059062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,128,0,1,float16,float16,0,0.05735999842484792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,128,0,1,float16,fp8,0,0.057461331288019814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,128,0,1,fp8,fp8,0,0.06435200075308482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,128,0,1,fp8,fp8,0,0.0709440012772878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,128,0,1,float16,float16,0,0.05936000247796377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,128,0,1,float16,fp8,0,0.05827199916044871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,128,0,1,float16,fp8,0,0.039034667114416756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,128,0,1,fp8,fp8,0,0.06647466619809468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,128,0,1,float16,float16,0,0.06645333270231883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,128,0,1,float16,fp8,0,0.06505600114663442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,128,0,1,float16,float16,0,0.04095999896526337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,128,0,1,fp8,fp8,0,0.03812266637881597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,128,0,1,fp8,fp8,0,0.04543466866016388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,128,0,1,float16,float16,0,0.033759998778502144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,128,0,1,fp8,fp8,0,0.04138666639725367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,128,0,1,float16,fp8,0,0.03425599883000056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,128,0,1,fp8,fp8,0,0.03745600084463755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,128,0,1,float16,float16,0,0.034847999612490334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,128,0,1,float16,fp8,0,0.03448000053564707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,128,0,1,float16,float16,0,0.037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,128,0,1,float16,fp8,0,0.023669332265853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,128,0,1,fp8,fp8,0,0.024682665864626568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,128,0,1,float16,fp8,0,0.03625600039958954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,128,0,1,float16,fp8,0,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,128,0,1,fp8,fp8,0,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,128,0,1,float16,fp8,0,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,128,0,1,float16,float16,0,0.02369066576162974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,128,0,1,float16,float16,0,0.023957334458827972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,128,0,1,float16,fp8,0,0.024080000817775726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,128,0,1,fp8,fp8,0,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,128,0,1,fp8,fp8,0,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,128,0,1,float16,float16,0,0.024671999116738636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,128,0,1,fp8,fp8,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,128,0,1,fp8,fp8,0,0.02611200014750163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,128,0,1,float16,fp8,0,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,128,0,1,float16,float16,0,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,128,0,1,float16,fp8,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,128,0,1,float16,float16,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,128,0,1,float16,fp8,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,128,0,1,fp8,fp8,0,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,128,0,1,float16,float16,0,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,128,0,1,float16,fp8,0,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,128,0,1,fp8,fp8,0,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,128,0,1,float16,float16,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,128,0,1,float16,fp8,0,0.016410666207472484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,128,0,1,fp8,fp8,0,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,128,0,1,float16,float16,0,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,128,0,1,float16,fp8,0,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,128,0,1,fp8,fp8,0,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,128,0,1,float16,float16,0,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,128,0,1,float16,float16,0,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,128,0,1,float16,fp8,0,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,128,0,1,fp8,fp8,0,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,128,0,1,float16,float16,0,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,128,0,1,float16,fp8,0,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,128,0,1,fp8,fp8,0,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,128,0,1,float16,fp8,0,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,128,0,1,fp8,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,128,0,1,float16,float16,0,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,128,0,1,float16,fp8,0,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,128,0,1,fp8,fp8,0,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,128,0,1,float16,float16,0,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,128,0,1,float16,fp8,0,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,128,0,1,fp8,fp8,0,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,128,0,1,float16,float16,0,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,128,0,1,float16,fp8,0,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,128,0,1,fp8,fp8,0,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,128,0,1,float16,fp8,0,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,128,0,1,fp8,fp8,0,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,128,0,1,float16,float16,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,128,0,1,float16,fp8,0,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,128,0,1,fp8,fp8,0,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,128,0,1,float16,float16,0,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,128,0,1,float16,fp8,0,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,128,0,1,fp8,fp8,0,0.016117333124081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,128,0,1,fp8,fp8,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,128,0,1,float16,float16,0,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,128,0,1,float16,fp8,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,128,0,1,float16,float16,0,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,128,0,1,float16,fp8,0,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,128,0,1,fp8,fp8,0,0.01618133361140887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,128,0,1,float16,fp8,0,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,128,0,1,float16,fp8,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,128,0,1,fp8,fp8,0,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,128,0,1,float16,float16,0,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,128,0,1,fp8,fp8,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,128,0,1,float16,float16,0,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,128,0,1,float16,fp8,0,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,128,0,1,fp8,fp8,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,128,0,1,float16,float16,0,0.0730506678422292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,128,0,1,fp8,fp8,0,0.09195199608802795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,128,0,1,float16,fp8,0,0.07196799914042155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,128,0,1,float16,float16,0,0.07533333202203114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,128,0,1,float16,fp8,0,0.07403733332951863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,128,0,1,fp8,fp8,0,0.0935093363126119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,128,0,1,float16,float16,0,0.08038933575153351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,128,0,1,float16,fp8,0,0.08032000064849854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,128,0,1,float16,fp8,0,0.04120533416668574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,128,0,1,float16,float16,0,0.04822933177153269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,128,0,1,fp8,fp8,0,0.10002133250236511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,128,0,1,fp8,fp8,0,0.052485331892967224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,128,0,1,float16,fp8,0,0.04638933142026266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,128,0,1,fp8,fp8,0,0.06020266811052958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,128,0,1,float16,float16,0,0.04163199911514918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,128,0,1,fp8,fp8,0,0.05524266759554545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,128,0,1,fp8,fp8,0,0.05120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,128,0,1,float16,float16,0,0.04176533222198486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,128,0,1,float16,fp8,0,0.04173333446184794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,128,0,1,float16,float16,0,0.043525333205858864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,128,0,1,float16,fp8,0,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,128,0,1,float16,float16,0,0.027935999135176342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,128,0,1,float16,fp8,0,0.028325334191322327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,128,0,1,fp8,fp8,0,0.03376533339420954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,128,0,1,float16,float16,0,0.026250667870044708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,128,0,1,float16,fp8,0,0.026501332720120747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,128,0,1,fp8,fp8,0,0.03224000086386999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,128,0,1,float16,float16,0,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,128,0,1,float16,fp8,0,0.02672533442576726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,128,0,1,fp8,fp8,0,0.032655999064445496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,128,0,1,float16,float16,0,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,128,0,1,float16,fp8,0,0.027776000400384266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,128,0,1,fp8,fp8,0,0.033173332611719765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,128,0,1,float16,float16,0,0.019834666202465694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,128,0,1,float16,fp8,0,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,128,0,1,fp8,fp8,0,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,128,0,1,float16,float16,0,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,128,0,1,float16,fp8,0,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,128,0,1,fp8,fp8,0,0.022170667846997578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,128,0,1,float16,fp8,0,0.01941866676012675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,128,0,1,float16,float16,0,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,128,0,1,float16,fp8,0,0.019199999670187633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,128,0,1,fp8,fp8,0,0.022463999688625336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,128,0,1,float16,float16,0,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,128,0,1,fp8,fp8,0,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,128,0,1,float16,float16,0,0.015685333559910457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,128,0,1,float16,fp8,0,0.01607999950647354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,128,0,1,fp8,fp8,0,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,128,0,1,float16,float16,0,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,128,0,1,float16,fp8,0,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,128,0,1,fp8,fp8,0,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,128,0,1,float16,float16,0,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,128,0,1,float16,fp8,0,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,128,0,1,fp8,fp8,0,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,128,0,1,float16,fp8,0,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,128,0,1,fp8,fp8,0,0.017946666727463405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,128,0,1,float16,float16,0,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,128,0,1,float16,fp8,0,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,128,0,1,fp8,fp8,0,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,128,0,1,float16,float16,0,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,128,0,1,float16,fp8,0,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,128,0,1,fp8,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,128,0,1,float16,float16,0,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,128,0,1,float16,fp8,0,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,128,0,1,fp8,fp8,0,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,128,0,1,float16,float16,0,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,128,0,1,float16,fp8,0,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,128,0,1,fp8,fp8,0,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,128,0,1,float16,float16,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,128,0,1,float16,fp8,0,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,128,0,1,fp8,fp8,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,128,0,1,float16,float16,0,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,128,0,1,float16,fp8,0,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,128,0,1,fp8,fp8,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,128,0,1,float16,float16,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,128,0,1,float16,fp8,0,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,128,0,1,fp8,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,128,0,1,float16,float16,0,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,128,0,1,float16,fp8,0,0.01422400027513504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,128,0,1,float16,fp8,0,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,128,0,1,fp8,fp8,0,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,128,0,1,fp8,fp8,0,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,128,0,1,float16,float16,0,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,128,0,1,float16,fp8,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,128,0,1,fp8,fp8,0,0.01634666696190834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,128,0,1,float16,float16,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,128,0,1,float16,float16,0,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,128,0,1,float16,fp8,0,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,128,0,1,fp8,fp8,0,0.016341333587964375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,128,0,1,float16,float16,0,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,128,0,1,float16,fp8,0,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,128,0,1,fp8,fp8,0,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,128,0,1,float16,float16,0,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,128,0,1,float16,fp8,0,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,128,0,1,float16,fp8,0,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,128,0,1,fp8,fp8,0,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,128,0,1,float16,float16,0,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,128,0,1,float16,fp8,0,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,128,0,1,float16,fp8,0,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,128,0,1,fp8,fp8,0,0.015989333391189575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,128,0,1,float16,float16,0,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,128,0,1,fp8,fp8,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,128,0,1,float16,float16,0,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,128,0,1,fp8,fp8,0,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,128,0,1,float16,float16,0,0.05982399980227152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,128,0,1,float16,fp8,0,0.05972800155480703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,128,0,1,fp8,fp8,0,0.07929599781831105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,128,0,1,fp8,fp8,0,0.08328533172607422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,128,0,1,float16,float16,0,0.06011199951171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,128,0,1,float16,fp8,0,0.036570665736993156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,128,0,1,fp8,fp8,0,0.04822400212287903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,128,0,1,float16,fp8,0,0.05996266504128774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,128,0,1,float16,fp8,0,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,128,0,1,fp8,fp8,0,0.08144000172615051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,128,0,1,float16,float16,0,0.0626933326323827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,128,0,1,float16,fp8,0,0.06262399752934773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,128,0,1,fp8,fp8,0,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,128,0,1,float16,float16,0,0.03719466676314672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,128,0,1,float16,float16,0,0.03738666574160258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,128,0,1,fp8,fp8,0,0.048538664976755776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,128,0,1,float16,float16,0,0.035904000202814736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,128,0,1,fp8,fp8,0,0.04708800216515859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,128,0,1,float16,float16,0,0.024245334168275196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,128,0,1,float16,float16,0,0.03583466758330663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,128,0,1,float16,fp8,0,0.03586666782697042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,128,0,1,float16,fp8,0,0.03659733384847641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,128,0,1,float16,float16,0,0.024666666984558105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,128,0,1,float16,fp8,0,0.024821333587169647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,128,0,1,fp8,fp8,0,0.03046400099992752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,128,0,1,float16,fp8,0,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,128,0,1,fp8,fp8,0,0.02977599948644638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,128,0,1,float16,float16,0,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,128,0,1,fp8,fp8,0,0.030373332401116688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,128,0,1,float16,fp8,0,0.02420799930890401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,128,0,1,fp8,fp8,0,0.02976000060637792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,128,0,1,float16,float16,0,0.024362665911515553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,128,0,1,float16,fp8,0,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,128,0,1,float16,fp8,0,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,128,0,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,128,0,1,float16,float16,0,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,128,0,1,float16,fp8,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,128,0,1,float16,fp8,0,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,128,0,1,fp8,fp8,0,0.02094399929046631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,128,0,1,float16,float16,0,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,128,0,1,float16,fp8,0,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,128,0,1,fp8,fp8,0,0.02160533269246419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,128,0,1,float16,float16,0,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,128,0,1,fp8,fp8,0,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,128,0,1,float16,float16,0,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,128,0,1,float16,fp8,0,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,128,0,1,fp8,fp8,0,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,128,0,1,float16,float16,0,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,128,0,1,float16,fp8,0,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,128,0,1,fp8,fp8,0,0.016613333175579708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,128,0,1,float16,float16,0,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,128,0,1,float16,fp8,0,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,128,0,1,fp8,fp8,0,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,128,0,1,float16,fp8,0,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,128,0,1,fp8,fp8,0,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,128,0,1,float16,float16,0,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,128,0,1,float16,fp8,0,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,128,0,1,fp8,fp8,0,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,128,0,1,float16,float16,0,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,128,0,1,float16,fp8,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,128,0,1,float16,fp8,0,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,128,0,1,fp8,fp8,0,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,128,0,1,fp8,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,128,0,1,float16,float16,0,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,128,0,1,float16,float16,0,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,128,0,1,float16,fp8,0,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,128,0,1,float16,fp8,0,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,128,0,1,fp8,fp8,0,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,128,0,1,float16,float16,0,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,128,0,1,float16,fp8,0,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,128,0,1,fp8,fp8,0,0.016000000139077503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,128,0,1,float16,float16,0,0.013631999492645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,128,0,1,fp8,fp8,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,128,0,1,float16,float16,0,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,128,0,1,float16,float16,0,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,128,0,1,float16,fp8,0,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,128,0,1,fp8,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,128,0,1,float16,float16,0,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,128,0,1,float16,fp8,0,0.014389333625634512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,128,0,1,fp8,fp8,0,0.01597333326935768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,128,0,1,float16,fp8,0,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,128,0,1,fp8,fp8,0,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,128,0,1,float16,float16,0,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,128,0,1,float16,fp8,0,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,128,0,1,fp8,fp8,0,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,128,0,1,float16,float16,0,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,128,0,1,float16,fp8,0,0.014085333794355392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,128,0,1,fp8,fp8,0,0.015930666277805965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,128,0,1,float16,float16,0,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,128,0,1,float16,fp8,0,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,128,0,1,fp8,fp8,0,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,128,0,1,float16,float16,0,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,128,0,1,float16,fp8,0,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,128,0,1,fp8,fp8,0,0.015685333559910457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,128,0,1,float16,float16,0,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,128,0,1,float16,float16,0,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,128,0,1,float16,fp8,0,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,128,0,1,fp8,fp8,0,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,128,0,1,float16,float16,0,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,128,0,1,float16,fp8,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,128,0,1,fp8,fp8,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,128,0,1,float16,fp8,0,0.014074667046467463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,128,0,1,fp8,fp8,0,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,128,0,1,fp8,fp8,0,2.6478187243143716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,128,0,1,float16,float16,0,4.369802792867024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,128,0,1,float16,float16,0,2.14245335261027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,128,0,1,fp8,fp8,0,2.6897013982137046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,128,0,1,float16,fp8,0,2.1546079317728677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,128,0,1,float16,fp8,0,4.326671918233235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,128,0,1,float16,float16,0,4.232042630513509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,128,0,1,float16,fp8,0,4.278143882751465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,128,0,1,fp8,fp8,0,1.4345547358194988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,128,0,1,float16,float16,0,2.0834506352742515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,128,0,1,fp8,fp8,0,1.403157393137614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,128,0,1,float16,fp8,0,2.066314697265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,128,0,1,fp8,fp8,0,1.451573371887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,128,0,1,float16,float16,0,1.1121386686960857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,128,0,1,float16,fp8,0,1.0939733187357585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,128,0,1,float16,float16,0,2.114960034688314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,128,0,1,float16,float16,0,1.0926132996877034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,128,0,1,float16,fp8,0,2.0775252978006997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,128,0,1,fp8,fp8,0,0.7062719662984213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,128,0,1,float16,float16,0,1.0973493258158367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,128,0,1,fp8,fp8,0,0.682207981745402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,128,0,1,float16,fp8,0,1.1162827014923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,128,0,1,float16,fp8,0,1.0787893136342366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,128,0,1,fp8,fp8,0,0.6766773064931234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,128,0,1,fp8,fp8,0,0.3606293201446533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,128,0,1,float16,float16,0,0.5101813475290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,128,0,1,float16,fp8,0,0.5084160168965658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,128,0,1,float16,float16,0,0.49833067258199054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,128,0,1,float16,fp8,0,0.4989653428395589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,128,0,1,fp8,fp8,0,0.35782400767008465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,128,0,1,float16,float16,0,0.49188800652821857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,128,0,1,fp8,fp8,0,0.3566666841506958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,128,0,1,float16,fp8,0,0.494432012240092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,128,0,1,fp8,fp8,0,1.6007359822591145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,128,0,1,float16,float16,0,1.2271253267923992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,128,0,1,float16,float16,0,2.423818588256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,128,0,1,float16,fp8,0,2.3113120396931968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,128,0,1,fp8,fp8,0,1.586634635925293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,128,0,1,float16,fp8,0,1.2224693298339844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,128,0,1,float16,float16,0,2.426250616709391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,128,0,1,fp8,fp8,0,0.841381311416626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,128,0,1,float16,fp8,0,2.4077332814534507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,128,0,1,float16,float16,0,1.2007839679718018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,128,0,1,float16,fp8,0,1.2188426653544109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,128,0,1,fp8,fp8,0,0.8315520286560059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,128,0,1,float16,float16,0,1.1996479829152424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,128,0,1,float16,float16,0,0.5797280073165894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,128,0,1,fp8,fp8,0,0.802133321762085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,128,0,1,fp8,fp8,0,0.42113598187764484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,128,0,1,float16,fp8,0,0.590122660001119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,128,0,1,float16,fp8,0,1.2095839977264404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,128,0,1,float16,float16,0,0.5636106729507446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,128,0,1,float16,float16,0,0.3261653383572896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,128,0,1,fp8,fp8,0,0.40356266498565674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,128,0,1,float16,fp8,0,0.5620799859364828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,128,0,1,float16,float16,0,0.5669120152791342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,128,0,1,fp8,fp8,0,0.4133760134379069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,128,0,1,float16,fp8,0,0.5635786851247152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,128,0,1,float16,fp8,0,0.337936004002889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,128,0,1,fp8,fp8,0,0.2435306708017985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,128,0,1,fp8,fp8,0,0.23233066002527872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,128,0,1,float16,float16,0,0.320303996404012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,128,0,1,float16,fp8,0,0.31623466809590656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,128,0,1,float16,float16,0,0.32520000139872235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,128,0,1,float16,fp8,0,0.3213119904200236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,128,0,1,fp8,fp8,0,1.16866668065389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,128,0,1,fp8,fp8,0,0.2378879984219869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,128,0,1,float16,float16,0,1.6510826746622722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,128,0,1,float16,fp8,0,1.6697440147399902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,128,0,1,float16,float16,0,1.6577547391255696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,128,0,1,float16,fp8,0,1.6798933347066243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,128,0,1,float16,float16,0,0.9036853313446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,128,0,1,fp8,fp8,0,1.1560959815979004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,128,0,1,float16,fp8,0,0.8926613330841064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,128,0,1,fp8,fp8,0,0.5935839811960856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,128,0,1,float16,float16,0,0.8776853084564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,128,0,1,float16,fp8,0,0.8716639677683512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,128,0,1,fp8,fp8,0,0.5852799812952677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,128,0,1,fp8,fp8,0,0.5740960041681925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,128,0,1,float16,float16,0,0.8907466729482015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,128,0,1,float16,float16,0,0.4262400070826213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,128,0,1,float16,fp8,0,0.42904531955718994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,128,0,1,fp8,fp8,0,0.31462399164835614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,128,0,1,float16,fp8,0,0.8789386749267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,128,0,1,float16,fp8,0,0.41316266854604083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,128,0,1,float16,float16,0,0.411077340443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,128,0,1,fp8,fp8,0,0.306005338827769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,128,0,1,float16,float16,0,0.41917868455251056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,128,0,1,float16,float16,0,0.23903467257817587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,128,0,1,float16,fp8,0,0.4204373359680176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,128,0,1,fp8,fp8,0,0.30922667185465497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,128,0,1,float16,fp8,0,0.23921066522598267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,128,0,1,fp8,fp8,0,0.17882132530212402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,128,0,1,float16,float16,0,0.23642667134602866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,128,0,1,float16,fp8,0,0.23641065756479898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,128,0,1,fp8,fp8,0,0.1749066710472107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,128,0,1,float16,float16,0,0.2373759945233663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,128,0,1,fp8,fp8,0,0.177839994430542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,128,0,1,float16,fp8,0,0.23819732666015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,128,0,1,fp8,fp8,0,1.5353867212931316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,128,0,1,float16,float16,0,2.2627199490865073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,128,0,1,float16,fp8,0,2.201338609059652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,128,0,1,float16,float16,0,2.2041865984598794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,128,0,1,float16,fp8,0,2.1504054069519043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,128,0,1,float16,float16,0,1.1396000385284424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,128,0,1,fp8,fp8,0,0.8292373021443685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,128,0,1,fp8,fp8,0,1.547525405883789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,128,0,1,float16,fp8,0,1.1440640290578206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,128,0,1,float16,float16,0,1.1157120068868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,128,0,1,fp8,fp8,0,0.8144373099009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,128,0,1,float16,fp8,0,1.1240479946136475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,128,0,1,float16,float16,0,1.138602654139201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,128,0,1,float16,fp8,0,1.117690642674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,128,0,1,fp8,fp8,0,0.8080320358276367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,128,0,1,float16,float16,0,0.6193120082219442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,128,0,1,float16,fp8,0,0.6158879995346069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,128,0,1,fp8,fp8,0,0.4084159930547078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,128,0,1,float16,float16,0,0.5961440006891886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,128,0,1,float16,fp8,0,0.5976106723149618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,128,0,1,fp8,fp8,0,0.38945599397023517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,128,0,1,float16,float16,0,0.6013760169347128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,128,0,1,float16,float16,0,0.28539733091990155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,128,0,1,fp8,fp8,0,0.3973173300425212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,128,0,1,float16,fp8,0,0.28551467259724933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,128,0,1,float16,fp8,0,0.6040106614430746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,128,0,1,fp8,fp8,0,0.21740265687306723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,128,0,1,float16,float16,0,0.2762613296508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,128,0,1,float16,fp8,0,0.27750933170318604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,128,0,1,fp8,fp8,0,0.20829866329828897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,128,0,1,float16,float16,0,0.2786986629168193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,128,0,1,float16,fp8,0,0.2807786663373311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,128,0,1,fp8,fp8,0,0.21056532859802246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,128,0,1,fp8,fp8,0,0.11312533418337505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,128,0,1,float16,float16,0,0.15507733821868896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,128,0,1,float16,fp8,0,0.15525333086649576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,128,0,1,fp8,fp8,0,0.12081066767374675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,128,0,1,float16,float16,0,0.1516800026098887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,128,0,1,float16,fp8,0,0.1511840025583903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,128,0,1,float16,float16,0,0.15222400426864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,128,0,1,float16,fp8,0,0.15313599507013956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,128,0,1,fp8,fp8,0,0.1160586675008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,128,0,1,fp8,fp8,0,0.9305013020833334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,128,0,1,float16,float16,0,1.3060426712036133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,128,0,1,float16,float16,0,1.2989599704742432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,128,0,1,float16,fp8,0,1.2825226783752441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,128,0,1,float16,float16,0,0.6933546861012777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,128,0,1,fp8,fp8,0,0.48558398087819415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,128,0,1,float16,fp8,0,0.684175968170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,128,0,1,fp8,fp8,0,0.9342933495839437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,128,0,1,float16,fp8,0,0.6987040042877197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,128,0,1,float16,fp8,0,1.2936586538950603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,128,0,1,fp8,fp8,0,0.4612106482187907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,128,0,1,float16,float16,0,0.6809706687927246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,128,0,1,float16,float16,0,0.6749707063039144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,128,0,1,fp8,fp8,0,0.47674667835235596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,128,0,1,float16,fp8,0,0.6754346688588461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,128,0,1,float16,float16,0,0.3347146511077881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,128,0,1,float16,fp8,0,0.3378293514251709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,128,0,1,fp8,fp8,0,0.25755733251571655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,128,0,1,float16,float16,0,0.3146880070368449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,128,0,1,float16,fp8,0,0.3211519916852315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,128,0,1,fp8,fp8,0,0.2423786719640096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,128,0,1,float16,float16,0,0.3222773273785909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,128,0,1,fp8,fp8,0,0.2478933334350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,128,0,1,float16,fp8,0,0.3223573366800944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,128,0,1,float16,float16,0,0.18774400154749551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,128,0,1,float16,fp8,0,0.18823466698328653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,128,0,1,fp8,fp8,0,0.1458346645037333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,128,0,1,float16,float16,0,0.1803413430849711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,128,0,1,float16,fp8,0,0.17997866868972778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,128,0,1,fp8,fp8,0,0.13925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,128,0,1,float16,float16,0,0.18257067600886026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,128,0,1,float16,fp8,0,0.1848319967587789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,128,0,1,fp8,fp8,0,0.14088533322016397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,128,0,1,float16,float16,0,0.11475732922554016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,128,0,1,float16,fp8,0,0.1158026655515035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,128,0,1,fp8,fp8,0,0.09018666545550029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,128,0,1,float16,float16,0,0.1141919990380605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,128,0,1,float16,fp8,0,0.11437867085138957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,128,0,1,fp8,fp8,0,0.08721066514650981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,128,0,1,float16,float16,0,0.11247467001279195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,128,0,1,float16,fp8,0,0.11476266384124756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,128,0,1,fp8,fp8,0,0.0864533285299937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,128,0,1,float16,float16,0,1.2684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,128,0,1,fp8,fp8,0,0.9413226445515951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,128,0,1,float16,fp8,0,1.2785066763559978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,128,0,1,fp8,fp8,0,0.5197493235270182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,128,0,1,float16,float16,0,0.6934933662414551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,128,0,1,float16,fp8,0,0.6773866812388102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,128,0,1,float16,float16,0,1.296890656153361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,128,0,1,fp8,fp8,0,0.9553120136260986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,128,0,1,float16,float16,0,0.651797334353129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,128,0,1,float16,fp8,0,1.286293347676595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,128,0,1,float16,fp8,0,0.6550613244374593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,128,0,1,fp8,fp8,0,0.48866132895151776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,128,0,1,float16,float16,0,0.6684746742248535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,128,0,1,fp8,fp8,0,0.5004586776097616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,128,0,1,float16,fp8,0,0.6561493476231893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,128,0,1,float16,float16,0,0.3651786645253499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,128,0,1,float16,fp8,0,0.3646986484527588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,128,0,1,fp8,fp8,0,0.26151466369628906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,128,0,1,float16,float16,0,0.34781865278879803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,128,0,1,float16,fp8,0,0.3455359935760498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,128,0,1,float16,fp8,0,0.17571733395258585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,128,0,1,fp8,fp8,0,0.23930666844050089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,128,0,1,float16,float16,0,0.352234681447347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,128,0,1,fp8,fp8,0,0.13177067041397095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,128,0,1,float16,fp8,0,0.34987199306488037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,128,0,1,fp8,fp8,0,0.24645866950352988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,128,0,1,float16,float16,0,0.17562667528788248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,128,0,1,fp8,fp8,0,0.1430400013923645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,128,0,1,float16,float16,0,0.16379732886950174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,128,0,1,float16,fp8,0,0.1648426651954651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,128,0,1,float16,float16,0,0.16671466827392578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,128,0,1,float16,fp8,0,0.16892266273498535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,128,0,1,fp8,fp8,0,0.13455999890963236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,128,0,1,float16,float16,0,0.09377599755922954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,128,0,1,float16,fp8,0,0.09476799766222636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,128,0,1,fp8,fp8,0,0.08029333253701527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,128,0,1,float16,float16,0,0.08924800157546997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,128,0,1,float16,fp8,0,0.08931733171145122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,128,0,1,fp8,fp8,0,0.07351466516653697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,128,0,1,float16,float16,0,0.09090666969617207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,128,0,1,float16,fp8,0,0.09103999535242717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,128,0,1,fp8,fp8,0,0.07520000139872234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,128,0,1,float16,float16,0,0.08054933448632558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,128,0,1,float16,fp8,0,0.08228800197442372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,128,0,1,fp8,fp8,0,0.06425599753856659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,128,0,1,float16,float16,0,0.07916800181070964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,128,0,1,float16,fp8,0,0.07938666641712189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,128,0,1,fp8,fp8,0,0.061941335598627724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,128,0,1,float16,float16,0,0.07973866661389668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,128,0,1,float16,fp8,0,0.08040000001589458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,128,0,1,fp8,fp8,0,0.06272533535957336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,128,0,1,float16,float16,0,0.7892853418986002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,128,0,1,fp8,fp8,0,0.5966933170954386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,128,0,1,float16,fp8,0,0.7818666299184164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,128,0,1,float16,float16,0,0.7962346871693929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,128,0,1,fp8,fp8,0,0.6118933359781901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,128,0,1,float16,fp8,0,0.8035840193430582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,128,0,1,float16,float16,0,0.40565331776936847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,128,0,1,float16,float16,0,0.43401598930358887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,128,0,1,float16,fp8,0,0.43197333812713623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,128,0,1,fp8,fp8,0,0.33478399117787677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,128,0,1,fp8,fp8,0,0.2993759910265605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,128,0,1,float16,fp8,0,0.4033386707305908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,128,0,1,float16,float16,0,0.4133439858754476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,128,0,1,float16,fp8,0,0.19491199652353922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,128,0,1,float16,fp8,0,0.41661866505940753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,128,0,1,fp8,fp8,0,0.31174399455388385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,128,0,1,float16,float16,0,0.21739733219146729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,128,0,1,float16,fp8,0,0.2158400019009908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,128,0,1,fp8,fp8,0,0.17250667015711466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,128,0,1,float16,float16,0,0.19503466288248697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,128,0,1,fp8,fp8,0,0.15691733360290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,128,0,1,float16,float16,0,0.20016000668207803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,128,0,1,float16,fp8,0,0.20564266045888266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,128,0,1,fp8,fp8,0,0.16168000300725302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,128,0,1,float16,float16,0,0.11642666657765706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,128,0,1,float16,fp8,0,0.11693867047627766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,128,0,1,fp8,fp8,0,0.09693333506584167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,128,0,1,float16,float16,0,0.10855467120806377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,128,0,1,float16,fp8,0,0.10878400007883708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,128,0,1,fp8,fp8,0,0.08779199918111165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,128,0,1,float16,float16,0,0.11062399546305339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,128,0,1,float16,fp8,0,0.1113866666952769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,128,0,1,fp8,fp8,0,0.09180266658465068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,128,0,1,float16,float16,0,0.06819733480612437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,128,0,1,float16,fp8,0,0.07011733452479045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,128,0,1,fp8,fp8,0,0.05986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,128,0,1,float16,float16,0,0.06710933148860931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,128,0,1,float16,fp8,0,0.06693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,128,0,1,fp8,fp8,0,0.053717335065205894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,128,0,1,float16,float16,0,0.06660266717274983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,128,0,1,float16,fp8,0,0.06748266518115997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,128,0,1,fp8,fp8,0,0.055173332492510475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,128,0,1,float16,fp8,0,0.06297066807746887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,128,0,1,float16,float16,0,0.06313600142796834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,128,0,1,float16,fp8,0,0.06382399797439575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,128,0,1,fp8,fp8,0,0.05102399984995524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,128,0,1,float16,fp8,0,0.06333333253860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,128,0,1,float16,float16,0,0.0626986672480901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,128,0,1,fp8,fp8,0,0.04980800052483877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,128,0,1,float16,float16,0,0.063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,128,0,1,fp8,fp8,0,0.04982399940490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,128,0,1,float16,float16,0,0.8216533660888672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,128,0,1,float16,fp8,0,0.8138453165690104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,128,0,1,fp8,fp8,0,0.6566666762034098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,128,0,1,float16,float16,0,0.46048001448313397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,128,0,1,float16,float16,0,0.837551991144816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,128,0,1,float16,fp8,0,0.831712007522583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,128,0,1,float16,fp8,0,0.451530655225118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,128,0,1,fp8,fp8,0,0.671066681543986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,128,0,1,fp8,fp8,0,0.3752693335215251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,128,0,1,float16,float16,0,0.4216373364130656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,128,0,1,float16,fp8,0,0.4219733476638794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,128,0,1,fp8,fp8,0,0.3407680193583171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,128,0,1,float16,float16,0,0.43355198701222736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,128,0,1,float16,fp8,0,0.4296640157699585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,128,0,1,fp8,fp8,0,0.34886399904886883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,128,0,1,float16,float16,0,0.24430400133132935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,128,0,1,float16,fp8,0,0.24573866526285806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,128,0,1,fp8,fp8,0,0.1928373376528422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,128,0,1,float16,float16,0,0.22209600607554117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,128,0,1,float16,fp8,0,0.22210667530695596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,128,0,1,fp8,fp8,0,0.17095466454823813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,128,0,1,float16,float16,0,0.22984532515207926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,128,0,1,float16,fp8,0,0.2300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,128,0,1,fp8,fp8,0,0.17810134092966715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,128,0,1,float16,float16,0,0.12230933705965678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,128,0,1,float16,fp8,0,0.12036266922950745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,128,0,1,fp8,fp8,0,0.10398933291435242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,128,0,1,float16,float16,0,0.10814932982126872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,128,0,1,fp8,fp8,0,0.09690133730570476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,128,0,1,float16,fp8,0,0.1085653305053711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,128,0,1,fp8,fp8,0,0.09289600451787312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,128,0,1,float16,float16,0,0.11100266377131145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,128,0,1,float16,fp8,0,0.11144000291824341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,128,0,1,float16,float16,0,0.06531199812889099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,128,0,1,float16,fp8,0,0.06577600042025249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,128,0,1,fp8,fp8,0,0.06151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,128,0,1,float16,float16,0,0.05973866581916809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,128,0,1,float16,fp8,0,0.059994667768478394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,128,0,1,fp8,fp8,0,0.052149335543314614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,128,0,1,float16,float16,0,0.06086933116118113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,128,0,1,float16,fp8,0,0.062133332093556724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,128,0,1,fp8,fp8,0,0.05461333195368449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,128,0,1,float16,float16,0,0.04956800242265066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,128,0,1,float16,fp8,0,0.05018133421738943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,128,0,1,fp8,fp8,0,0.04264533519744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,128,0,1,float16,float16,0,0.04874666531880697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,128,0,1,float16,fp8,0,0.04900266726811727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,128,0,1,fp8,fp8,0,0.04041066765785217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,128,0,1,float16,float16,0,0.049226666490236916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,128,0,1,float16,fp8,0,0.04931733508904775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,128,0,1,fp8,fp8,0,0.041221333046754204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,128,0,1,float16,float16,0,0.04599999884764353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,128,0,1,float16,float16,0,0.0462719996770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,128,0,1,float16,fp8,0,0.0473333348830541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,128,0,1,fp8,fp8,0,0.03821333249409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,128,0,1,float16,float16,0,0.045754666129748024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,128,0,1,float16,fp8,0,0.046207999189694725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,128,0,1,fp8,fp8,0,0.03766933331886927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,128,0,1,float16,fp8,0,0.0469813346862793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,128,0,1,fp8,fp8,0,0.03788800040880839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,128,0,1,float16,float16,0,0.513861338297526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,128,0,1,float16,fp8,0,0.5193493366241455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,128,0,1,fp8,fp8,0,0.4333653450012207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,128,0,1,float16,float16,0,0.5260159969329834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,128,0,1,float16,fp8,0,0.525381326675415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,128,0,1,fp8,fp8,0,0.4516106843948364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,128,0,1,float16,float16,0,0.30472532908121747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,128,0,1,fp8,fp8,0,0.214848001797994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,128,0,1,float16,fp8,0,0.2993920048077901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,128,0,1,fp8,fp8,0,0.26898133754730225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,128,0,1,float16,float16,0,0.27517332633336383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,128,0,1,float16,fp8,0,0.27105067173639935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,128,0,1,float16,float16,0,0.27990400791168213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,128,0,1,float16,fp8,0,0.27582399050394696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,128,0,1,float16,float16,0,0.16260799765586853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,128,0,1,fp8,fp8,0,0.22562666734059653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,128,0,1,float16,fp8,0,0.16208533445994058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,128,0,1,fp8,fp8,0,0.13316266735394797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,128,0,1,float16,float16,0,0.1343839963277181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,128,0,1,float16,fp8,0,0.13497066497802734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,128,0,1,fp8,fp8,0,0.11748266220092773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,128,0,1,float16,float16,0,0.14045866330464682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,128,0,1,fp8,fp8,0,0.0754559983809789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,128,0,1,float16,fp8,0,0.14059199889500937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,128,0,1,fp8,fp8,0,0.12242133418718974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,128,0,1,float16,fp8,0,0.07420800129572551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,128,0,1,float16,float16,0,0.0827946662902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,128,0,1,float16,fp8,0,0.08297599852085114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,128,0,1,float16,float16,0,0.07411733269691467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,128,0,1,fp8,fp8,0,0.06608533362547557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,128,0,1,float16,float16,0,0.07611200213432312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,128,0,1,float16,fp8,0,0.07790400087833405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,128,0,1,float16,float16,0,0.04614933331807455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,128,0,1,fp8,fp8,0,0.06924266616503398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,128,0,1,float16,float16,0,0.04457599918047587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,128,0,1,float16,fp8,0,0.0467199981212616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,128,0,1,fp8,fp8,0,0.0439573327700297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,128,0,1,float16,float16,0,0.04058133314053217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,128,0,1,float16,float16,0,0.04438933233420054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,128,0,1,float16,fp8,0,0.043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,128,0,1,float16,fp8,0,0.04035733391841253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,128,0,1,fp8,fp8,0,0.03869866579771042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,128,0,1,fp8,fp8,0,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,128,0,1,float16,fp8,0,0.04451733330885569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,128,0,1,fp8,fp8,0,0.04009066770474116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,128,0,1,float16,fp8,0,0.041146665811538696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,128,0,1,fp8,fp8,0,0.03491200009981791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,128,0,1,float16,float16,0,0.04037333279848099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,128,0,1,float16,float16,0,0.04071466624736786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,128,0,1,float16,fp8,0,0.04118400067090988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,128,0,1,fp8,fp8,0,0.03419200082619985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,128,0,1,float16,float16,0,0.03850133220354716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,128,0,1,float16,fp8,0,0.03851733356714249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,128,0,1,float16,fp8,0,0.03886933376391729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,128,0,1,fp8,fp8,0,0.03216533362865448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,128,0,1,float16,float16,0,0.03757333258787791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,128,0,1,float16,fp8,0,0.03833066672086716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,128,0,1,fp8,fp8,0,0.03177600105603536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,128,0,1,float16,float16,0,0.03811733424663544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,128,0,1,fp8,fp8,0,0.03224000086386999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,128,0,1,float16,float16,0,0.5533599853515625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,128,0,1,float16,fp8,0,0.5503626664479574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,128,0,1,fp8,fp8,0,0.4664320151011149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,128,0,1,float16,float16,0,0.5741866827011108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,128,0,1,float16,fp8,0,0.566927989323934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,128,0,1,fp8,fp8,0,0.474832018216451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,128,0,1,float16,float16,0,0.3349920113881429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,128,0,1,float16,fp8,0,0.32738665739695233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,128,0,1,fp8,fp8,0,0.2769706646601359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,128,0,1,float16,float16,0,0.2879040042559306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,128,0,1,float16,fp8,0,0.2890506585439046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,128,0,1,fp8,fp8,0,0.24471465746561685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,128,0,1,float16,float16,0,0.30080533027648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,128,0,1,float16,fp8,0,0.2967680096626282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,128,0,1,fp8,fp8,0,0.24796799818674722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,128,0,1,float16,float16,0,0.18150399128595987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,128,0,1,float16,fp8,0,0.16184533635775247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,128,0,1,float16,fp8,0,0.17731734116872153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,128,0,1,fp8,fp8,0,0.1444000005722046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,128,0,1,float16,float16,0,0.15752533078193665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,128,0,1,float16,fp8,0,0.15596266587575278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,128,0,1,fp8,fp8,0,0.12110933661460876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,128,0,1,float16,float16,0,0.16346133748690286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,128,0,1,fp8,fp8,0,0.12484266360600789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,128,0,1,float16,float16,0,0.09596799810727437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,128,0,1,float16,fp8,0,0.09330667058626811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,128,0,1,fp8,fp8,0,0.07932266592979431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,128,0,1,float16,float16,0,0.07830399771531422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,128,0,1,fp8,fp8,0,0.0702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,128,0,1,float16,fp8,0,0.07814933359622955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,128,0,1,fp8,fp8,0,0.06632533172766368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,128,0,1,float16,float16,0,0.08187733093897502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,128,0,1,float16,fp8,0,0.08167999982833862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,128,0,1,float16,float16,0,0.049312000473340355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,128,0,1,float16,fp8,0,0.048826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,128,0,1,fp8,fp8,0,0.04506133496761322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,128,0,1,float16,float16,0,0.043605332573254905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,128,0,1,float16,fp8,0,0.04332800209522247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,128,0,1,fp8,fp8,0,0.03696000079313914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,128,0,1,float16,float16,0,0.04446400205294291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,128,0,1,float16,fp8,0,0.044863998889923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,128,0,1,fp8,fp8,0,0.039477333426475525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,128,0,1,float16,float16,0,0.034101332227389015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,128,0,1,float16,fp8,0,0.0335413341720899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,128,0,1,fp8,fp8,0,0.02941333254178365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,128,0,1,float16,float16,0,0.03265066693226496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,128,0,1,float16,fp8,0,0.031914666295051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,128,0,1,fp8,fp8,0,0.02757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,128,0,1,float16,float16,0,0.03282133241494497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,128,0,1,float16,fp8,0,0.03305600086847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,128,0,1,fp8,fp8,0,0.02828266719977061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,128,0,1,float16,float16,0,0.030266667405764263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,128,0,1,float16,fp8,0,0.03062933435042699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,128,0,1,fp8,fp8,0,0.02603200078010559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,128,0,1,float16,float16,0,0.029215998947620392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,128,0,1,float16,fp8,0,0.029658667743206024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,128,0,1,fp8,fp8,0,0.025146665672461193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,128,0,1,float16,float16,0,0.029711998999118805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,128,0,1,float16,fp8,0,0.030181333422660828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,128,0,1,fp8,fp8,0,0.025472000241279602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,128,0,1,fp8,fp8,0,0.024127999941507976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,128,0,1,float16,float16,0,0.02845866729815801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,128,0,1,float16,fp8,0,0.02897600084543228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,128,0,1,fp8,fp8,0,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,128,0,1,float16,float16,0,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,128,0,1,float16,fp8,0,0.028304000695546467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,128,0,1,float16,float16,0,0.028373333315054577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,128,0,1,float16,fp8,0,0.02922133356332779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,128,0,1,fp8,fp8,0,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,128,0,1,float16,float16,0,0.44838400681813556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,128,0,1,float16,fp8,0,0.44660266240437824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,128,0,1,fp8,fp8,0,0.3808373212814331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,128,0,1,float16,float16,0,0.47419734795888263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,128,0,1,float16,float16,0,0.28134934107462567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,128,0,1,float16,fp8,0,0.46344534556070965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,128,0,1,fp8,fp8,0,0.3940426508585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,128,0,1,float16,fp8,0,0.2741706569989522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,128,0,1,fp8,fp8,0,0.23594133059183756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,128,0,1,float16,float16,0,0.23326400915781656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,128,0,1,float16,fp8,0,0.2327466607093811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,128,0,1,fp8,fp8,0,0.20001065731048584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,128,0,1,float16,float16,0,0.2462773323059082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,128,0,1,float16,fp8,0,0.2437866727511088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,128,0,1,fp8,fp8,0,0.20515199502309164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,128,0,1,float16,float16,0,0.15127467115720114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,128,0,1,float16,fp8,0,0.14774399995803833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,128,0,1,fp8,fp8,0,0.12563733259836832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,128,0,1,float16,float16,0,0.1262933313846588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,128,0,1,float16,fp8,0,0.12618666887283325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,128,0,1,fp8,fp8,0,0.10329066713651021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,128,0,1,float16,float16,0,0.13405332962671915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,128,0,1,float16,fp8,0,0.1321386694908142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,128,0,1,fp8,fp8,0,0.10565333565076192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,128,0,1,float16,float16,0,0.0867786705493927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,128,0,1,float16,fp8,0,0.08443733056386311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,128,0,1,fp8,fp8,0,0.07141333321730296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,128,0,1,float16,float16,0,0.06638399759928386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,128,0,1,float16,fp8,0,0.06606400012969971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,128,0,1,fp8,fp8,0,0.058245331048965454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,128,0,1,float16,float16,0,0.07102400064468384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,128,0,1,float16,fp8,0,0.0706826647122701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,128,0,1,float16,fp8,0,0.03738666574160258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,128,0,1,fp8,fp8,0,0.060533334811528526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,128,0,1,float16,float16,0,0.04397333165009817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,128,0,1,float16,fp8,0,0.03806933263937632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,128,0,1,float16,fp8,0,0.04262933135032654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,128,0,1,fp8,fp8,0,0.04203199843565623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,128,0,1,float16,float16,0,0.03701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,128,0,1,fp8,fp8,0,0.03288000077009201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,128,0,1,float16,float16,0,0.03826666623353958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,128,0,1,fp8,fp8,0,0.03563733398914337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,128,0,1,float16,float16,0,0.02794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,128,0,1,float16,fp8,0,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,128,0,1,fp8,fp8,0,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,128,0,1,float16,fp8,0,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,128,0,1,fp8,fp8,0,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,128,0,1,float16,float16,0,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,128,0,1,float16,fp8,0,0.026687999566396076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,128,0,1,fp8,fp8,0,0.02378133436044057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,128,0,1,float16,float16,0,0.023311999936898548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,128,0,1,float16,fp8,0,0.023786666492621105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,128,0,1,fp8,fp8,0,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,128,0,1,float16,float16,0,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,128,0,1,float16,fp8,0,0.022789334257443745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,128,0,1,fp8,fp8,0,0.02042666698495547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,128,0,1,float16,float16,0,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,128,0,1,float16,fp8,0,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,128,0,1,fp8,fp8,0,0.02090666691462199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,128,0,1,float16,float16,0,0.021914665897687275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,128,0,1,float16,fp8,0,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,128,0,1,fp8,fp8,0,0.019978666057189304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,128,0,1,float16,float16,0,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,128,0,1,float16,fp8,0,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,128,0,1,fp8,fp8,0,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,128,0,1,float16,float16,0,0.02049066623051961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,128,0,1,float16,float16,0,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,128,0,1,float16,fp8,0,0.022448000808556873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,128,0,1,fp8,fp8,0,0.019685332973798115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,128,0,1,float16,float16,0,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,128,0,1,float16,fp8,0,0.02239999920129776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,128,0,1,fp8,fp8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,128,0,1,float16,fp8,0,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,128,0,1,fp8,fp8,0,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,128,0,1,float16,float16,0,0.02205866575241089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,128,0,1,float16,fp8,0,0.02235200007756551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,128,0,1,fp8,fp8,0,0.16458666324615479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,128,0,1,fp8,fp8,0,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,128,0,1,float16,float16,0,0.2040533423423767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,128,0,1,float16,fp8,0,0.2031360069910685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,128,0,1,float16,float16,0,0.22014933824539185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,128,0,1,float16,fp8,0,0.2144320011138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,128,0,1,fp8,fp8,0,0.17441066106160483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,128,0,1,float16,float16,0,0.14430933197339377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,128,0,1,float16,fp8,0,0.13941333691279092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,128,0,1,fp8,fp8,0,0.11678399642308553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,128,0,1,float16,float16,0,0.11318399508794148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,128,0,1,float16,fp8,0,0.11191466450691223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,128,0,1,fp8,fp8,0,0.09303999940554301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,128,0,1,float16,float16,0,0.12100799878438313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,128,0,1,float16,fp8,0,0.11946133772532146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,128,0,1,fp8,fp8,0,0.09769066174825032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,128,0,1,float16,float16,0,0.0828906645377477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,128,0,1,float16,fp8,0,0.07992533346017201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,128,0,1,fp8,fp8,0,0.06727466483910878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,128,0,1,float16,float16,0,0.06125866870085398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,128,0,1,float16,fp8,0,0.0598826656738917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,128,0,1,fp8,fp8,0,0.05264533559481303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,128,0,1,float16,float16,0,0.06644799808661143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,128,0,1,float16,fp8,0,0.06470400094985962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,128,0,1,fp8,fp8,0,0.05731200178464254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,128,0,1,float16,float16,0,0.04010133445262909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,128,0,1,float16,fp8,0,0.03416533271471659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,128,0,1,float16,fp8,0,0.0391893337170283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,128,0,1,fp8,fp8,0,0.038015998899936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,128,0,1,float16,float16,0,0.034074666599432625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,128,0,1,fp8,fp8,0,0.03044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,128,0,1,float16,float16,0,0.035589332381884255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,128,0,1,float16,fp8,0,0.036117332677046456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,128,0,1,fp8,fp8,0,0.032698666055997215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,128,0,1,float16,float16,0,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,128,0,1,float16,fp8,0,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,128,0,1,fp8,fp8,0,0.022613334159056347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,128,0,1,float16,float16,0,0.024538666009902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,128,0,1,float16,fp8,0,0.024069334069887798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,128,0,1,fp8,fp8,0,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,128,0,1,float16,float16,0,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,128,0,1,float16,float16,0,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,128,0,1,float16,fp8,0,0.019567999988794327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,128,0,1,float16,fp8,0,0.024490666886170704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,128,0,1,fp8,fp8,0,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,128,0,1,float16,float16,0,0.020448000480731327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,128,0,1,float16,fp8,0,0.02060266708334287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,128,0,1,fp8,fp8,0,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,128,0,1,fp8,fp8,0,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,128,0,1,float16,float16,0,0.019952000429232914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,128,0,1,float16,fp8,0,0.020037333170572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,128,0,1,fp8,fp8,0,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,128,0,1,float16,float16,0,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,128,0,1,float16,fp8,0,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,128,0,1,fp8,fp8,0,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,128,0,1,float16,float16,0,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,128,0,1,float16,fp8,0,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,128,0,1,fp8,fp8,0,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,128,0,1,float16,float16,0,0.0185759998857975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,128,0,1,float16,fp8,0,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,128,0,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,128,0,1,float16,float16,0,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,128,0,1,float16,float16,0,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,128,0,1,float16,fp8,0,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,128,0,1,float16,float16,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,128,0,1,float16,fp8,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,128,0,1,float16,fp8,0,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,128,0,1,fp8,fp8,0,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,128,0,1,float16,float16,0,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,128,0,1,float16,fp8,0,0.018165333817402523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,128,0,1,fp8,fp8,0,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,128,0,1,fp8,fp8,0,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,128,0,1,fp8,fp8,0,0.016074666132529575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,128,0,1,float16,float16,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,128,0,1,float16,fp8,0,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,128,0,1,fp8,fp8,0,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,128,0,1,float16,float16,0,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,128,0,1,float16,fp8,0,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,128,0,1,fp8,fp8,0,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,128,0,1,float16,fp8,0,0.11965333422025044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,128,0,1,float16,float16,0,0.11107200384140015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,128,0,1,float16,fp8,0,0.11014399925867717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,128,0,1,fp8,fp8,0,0.0916426678498586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,128,0,1,float16,float16,0,0.12106133500734965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,128,0,1,fp8,fp8,0,0.09685333569844563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,128,0,1,float16,float16,0,0.07832533121109009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,128,0,1,float16,float16,0,0.06358399987220764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,128,0,1,float16,fp8,0,0.07418666779994965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,128,0,1,fp8,fp8,0,0.06454933186372121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,128,0,1,float16,float16,0,0.056474665800730385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,128,0,1,float16,fp8,0,0.05566399792830149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,128,0,1,fp8,fp8,0,0.052943999568621315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,128,0,1,float16,fp8,0,0.06198399762312571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,128,0,1,fp8,fp8,0,0.056186666091283165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,128,0,1,float16,float16,0,0.039647998909155525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,128,0,1,float16,fp8,0,0.037733333806196846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,128,0,1,fp8,fp8,0,0.03807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,128,0,1,fp8,fp8,0,0.031770666440327965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,128,0,1,float16,float16,0,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,128,0,1,float16,fp8,0,0.03299200038115183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,128,0,1,fp8,fp8,0,0.03036266565322876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,128,0,1,float16,float16,0,0.03474666674931844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,128,0,1,float16,fp8,0,0.03522666543722153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,128,0,1,float16,float16,0,0.024330665667851765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,128,0,1,float16,float16,0,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,128,0,1,float16,fp8,0,0.024245334168275196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,128,0,1,fp8,fp8,0,0.02236266682545344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,128,0,1,float16,float16,0,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,128,0,1,float16,fp8,0,0.022831998765468597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,128,0,1,fp8,fp8,0,0.021573332448800404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,128,0,1,float16,fp8,0,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,128,0,1,fp8,fp8,0,0.02205866575241089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,128,0,1,float16,float16,0,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,128,0,1,float16,fp8,0,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,128,0,1,float16,fp8,0,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,128,0,1,fp8,fp8,0,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,128,0,1,float16,float16,0,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,128,0,1,float16,fp8,0,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,128,0,1,fp8,fp8,0,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,128,0,1,float16,float16,0,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,128,0,1,fp8,fp8,0,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,128,0,1,float16,float16,0,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,128,0,1,float16,float16,0,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,128,0,1,float16,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,128,0,1,fp8,fp8,0,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,128,0,1,float16,float16,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,128,0,1,float16,fp8,0,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,128,0,1,float16,float16,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,128,0,1,fp8,fp8,0,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,128,0,1,float16,fp8,0,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,128,0,1,fp8,fp8,0,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,128,0,1,float16,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,128,0,1,float16,float16,0,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,128,0,1,float16,fp8,0,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,128,0,1,float16,float16,0,0.016000000139077503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,128,0,1,fp8,fp8,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,128,0,1,float16,fp8,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,128,0,1,fp8,fp8,0,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,128,0,1,float16,float16,0,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,128,0,1,fp8,fp8,0,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,128,0,1,float16,fp8,0,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,128,0,1,fp8,fp8,0,0.01599466676513354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,128,0,1,float16,float16,0,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,128,0,1,float16,fp8,0,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,128,0,1,fp8,fp8,0,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,128,0,1,float16,float16,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,128,0,1,float16,fp8,0,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,128,0,1,fp8,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,128,0,1,float16,float16,0,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,128,0,1,float16,fp8,0,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,128,0,1,fp8,fp8,0,0.0162773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,128,0,1,float16,float16,0,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,128,0,1,float16,fp8,0,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,128,0,1,fp8,fp8,0,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,128,0,1,float16,float16,0,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,128,0,1,float16,fp8,0,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,128,0,1,fp8,fp8,0,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,128,0,1,float16,float16,0,0.05880533158779144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,128,0,1,float16,fp8,0,0.05806399881839752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,128,0,1,fp8,fp8,0,0.06692266464233398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,128,0,1,float16,float16,0,0.06628266473611195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,128,0,1,float16,fp8,0,0.0643039991458257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,128,0,1,fp8,fp8,0,0.0717493345340093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,128,0,1,float16,float16,0,0.04048533240954081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,128,0,1,float16,fp8,0,0.039274667700131737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,128,0,1,fp8,fp8,0,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,128,0,1,float16,float16,0,0.03446933378775915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,128,0,1,float16,fp8,0,0.034128000338872276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,128,0,1,fp8,fp8,0,0.037776000797748566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,128,0,1,float16,float16,0,0.036202666660149894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,128,0,1,float16,fp8,0,0.03633599976698557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,128,0,1,fp8,fp8,0,0.04061333338419596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,128,0,1,float16,fp8,0,0.02386133372783661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,128,0,1,float16,float16,0,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,128,0,1,float16,fp8,0,0.024847999215126038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,128,0,1,fp8,fp8,0,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,128,0,1,float16,float16,0,0.02367999901374181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,128,0,1,fp8,fp8,0,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,128,0,1,float16,float16,0,0.02418133368094762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,128,0,1,float16,float16,0,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,128,0,1,float16,fp8,0,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,128,0,1,float16,fp8,0,0.02426133304834366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,128,0,1,fp8,fp8,0,0.024901332954565685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,128,0,1,float16,float16,0,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,128,0,1,float16,fp8,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,128,0,1,fp8,fp8,0,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,128,0,1,float16,fp8,0,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,128,0,1,fp8,fp8,0,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,128,0,1,float16,float16,0,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,128,0,1,float16,fp8,0,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,128,0,1,float16,float16,0,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,128,0,1,fp8,fp8,0,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,128,0,1,float16,float16,0,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,128,0,1,fp8,fp8,0,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,128,0,1,float16,float16,0,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,128,0,1,float16,fp8,0,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,128,0,1,fp8,fp8,0,0.016693333784739178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,128,0,1,float16,fp8,0,0.016250666230916977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,128,0,1,fp8,fp8,0,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,128,0,1,float16,float16,0,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,128,0,1,float16,fp8,0,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,128,0,1,fp8,fp8,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,128,0,1,float16,float16,0,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,128,0,1,float16,fp8,0,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,128,0,1,fp8,fp8,0,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,128,0,1,float16,float16,0,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,128,0,1,float16,fp8,0,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,128,0,1,fp8,fp8,0,0.016410666207472484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,128,0,1,float16,float16,0,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,128,0,1,float16,fp8,0,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,128,0,1,fp8,fp8,0,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,128,0,1,float16,float16,0,0.014490666488806406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,128,0,1,float16,fp8,0,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,128,0,1,fp8,fp8,0,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,128,0,1,float16,float16,0,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,128,0,1,float16,fp8,0,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,128,0,1,float16,float16,0,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,128,0,1,fp8,fp8,0,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,128,0,1,float16,float16,0,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,128,0,1,float16,fp8,0,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,128,0,1,fp8,fp8,0,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,128,0,1,float16,fp8,0,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,128,0,1,fp8,fp8,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,128,0,1,float16,float16,0,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,128,0,1,float16,fp8,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,128,0,1,fp8,fp8,0,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,128,0,1,float16,fp8,0,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,128,0,1,fp8,fp8,0,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,128,0,1,float16,float16,0,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,128,0,1,float16,fp8,0,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,128,0,1,fp8,fp8,0,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,128,0,1,float16,float16,0,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,128,0,1,float16,fp8,0,0.042170668641726174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,128,0,1,float16,fp8,0,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,128,0,1,float16,fp8,0,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,128,0,1,fp8,fp8,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,128,0,1,float16,float16,0,0.04187199970086416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,128,0,1,float16,fp8,0,0.028181334336598713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,128,0,1,fp8,fp8,0,0.052298665046691895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,128,0,1,float16,float16,0,0.04341333111127218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,128,0,1,float16,fp8,0,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,128,0,1,fp8,fp8,0,0.053642665346463524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,128,0,1,float16,float16,0,0.02868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,128,0,1,fp8,fp8,0,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,128,0,1,float16,float16,0,0.026895999908447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,128,0,1,fp8,fp8,0,0.03265066693226496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,128,0,1,float16,float16,0,0.01941866676012675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,128,0,1,float16,float16,0,0.027509334186712902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,128,0,1,float16,fp8,0,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,128,0,1,fp8,fp8,0,0.03256533294916153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,128,0,1,float16,float16,0,0.01956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,128,0,1,float16,float16,0,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,128,0,1,float16,fp8,0,0.020128000527620316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,128,0,1,fp8,fp8,0,0.02329600105683009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,128,0,1,float16,fp8,0,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,128,0,1,fp8,fp8,0,0.02231466770172119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,128,0,1,float16,fp8,0,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,128,0,1,fp8,fp8,0,0.02231466770172119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,128,0,1,float16,float16,0,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,128,0,1,float16,fp8,0,0.01595199977358182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,128,0,1,fp8,fp8,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,128,0,1,float16,float16,0,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,128,0,1,float16,fp8,0,0.015568000574906668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,128,0,1,fp8,fp8,0,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,128,0,1,float16,float16,0,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,128,0,1,float16,float16,0,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,128,0,1,float16,fp8,0,0.01597333326935768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,128,0,1,fp8,fp8,0,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,128,0,1,float16,float16,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,128,0,1,float16,fp8,0,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,128,0,1,fp8,fp8,0,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,128,0,1,float16,fp8,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,128,0,1,fp8,fp8,0,0.0164533331990242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,128,0,1,float16,float16,0,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,128,0,1,float16,fp8,0,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,128,0,1,fp8,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,128,0,1,float16,float16,0,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,128,0,1,float16,float16,0,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,128,0,1,float16,fp8,0,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,128,0,1,fp8,fp8,0,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,128,0,1,float16,float16,0,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,128,0,1,float16,fp8,0,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,128,0,1,fp8,fp8,0,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,128,0,1,float16,fp8,0,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,128,0,1,fp8,fp8,0,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,128,0,1,float16,float16,0,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,128,0,1,float16,fp8,0,0.014554666976133982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,128,0,1,fp8,fp8,0,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,128,0,1,float16,float16,0,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,128,0,1,float16,fp8,0,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,128,0,1,float16,fp8,0,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,128,0,1,fp8,fp8,0,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,128,0,1,float16,float16,0,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,128,0,1,float16,fp8,0,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,128,0,1,fp8,fp8,0,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,128,0,1,float16,float16,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,128,0,1,fp8,fp8,0,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,128,0,1,float16,float16,0,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,128,0,1,float16,fp8,0,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,128,0,1,fp8,fp8,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,128,0,1,float16,float16,0,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,128,0,1,float16,fp8,0,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,128,0,1,fp8,fp8,0,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,128,0,1,float16,float16,0,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,128,0,1,float16,fp8,0,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,128,0,1,fp8,fp8,0,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,128,0,1,float16,float16,0,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,128,0,1,float16,fp8,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,128,0,1,fp8,fp8,0,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,128,0,1,float16,float16,0,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,128,0,1,float16,fp8,0,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,128,0,1,fp8,fp8,0,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,128,0,1,float16,float16,0,0.03608533243338267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,128,0,1,float16,fp8,0,0.03605333218971888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,128,0,1,fp8,fp8,0,0.04704533517360687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,128,0,1,float16,float16,0,0.03690666705369949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,128,0,1,float16,fp8,0,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,128,0,1,fp8,fp8,0,0.047498668233553566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,128,0,1,float16,float16,0,0.024608001112937927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,128,0,1,float16,fp8,0,0.024383999407291412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,128,0,1,fp8,fp8,0,0.0305173322558403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,128,0,1,float16,float16,0,0.02366400013367335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,128,0,1,float16,fp8,0,0.023845332364241283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,128,0,1,fp8,fp8,0,0.029743999242782593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,128,0,1,float16,float16,0,0.024149333437283833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,128,0,1,float16,fp8,0,0.02426133304834366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,128,0,1,fp8,fp8,0,0.030058667063713074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,128,0,1,float16,float16,0,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,128,0,1,float16,fp8,0,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,128,0,1,fp8,fp8,0,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,128,0,1,float16,float16,0,0.01773333301146825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,128,0,1,float16,fp8,0,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,128,0,1,fp8,fp8,0,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,128,0,1,float16,float16,0,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,128,0,1,float16,fp8,0,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,128,0,1,float16,float16,0,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,128,0,1,fp8,fp8,0,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,128,0,1,float16,float16,0,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,128,0,1,float16,fp8,0,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,128,0,1,fp8,fp8,0,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,128,0,1,float16,fp8,0,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,128,0,1,fp8,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,128,0,1,float16,float16,0,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,128,0,1,float16,fp8,0,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,128,0,1,fp8,fp8,0,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,128,0,1,float16,float16,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,128,0,1,float16,fp8,0,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,128,0,1,fp8,fp8,0,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,128,0,1,fp8,fp8,0,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,128,0,1,float16,float16,0,0.014309333016475042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,128,0,1,float16,fp8,0,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,128,0,1,fp8,fp8,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,128,0,1,float16,float16,0,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,128,0,1,float16,fp8,0,0.014682666709025701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,128,0,1,float16,float16,0,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,128,0,1,float16,fp8,0,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,128,0,1,fp8,fp8,0,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,128,0,1,float16,float16,0,0.014229333649079004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,128,0,1,float16,fp8,0,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,128,0,1,fp8,fp8,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,128,0,1,float16,float16,0,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,128,0,1,float16,fp8,0,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,128,0,1,fp8,fp8,0,0.015861333658297855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,128,0,1,fp8,fp8,0,0.01605333387851715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,128,0,1,float16,float16,0,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,128,0,1,float16,fp8,0,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,128,0,1,fp8,fp8,0,0.016362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,128,0,1,float16,float16,0,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,128,0,1,float16,fp8,0,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,128,0,1,float16,float16,0,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,128,0,1,float16,fp8,0,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,128,0,1,fp8,fp8,0,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,128,0,1,fp8,fp8,0,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,128,0,1,float16,fp8,0,0.01458666721979777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,128,0,1,fp8,fp8,0,0.01579733317097028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,128,0,1,float16,float16,0,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,128,0,1,float16,fp8,0,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,128,0,1,float16,float16,0,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,128,0,1,float16,fp8,0,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,128,0,1,fp8,fp8,0,0.015893333901961643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,128,0,1,float16,float16,0,0.013946666071812311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,128,0,1,float16,fp8,0,0.014357333381970724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,128,0,1,fp8,fp8,0,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,128,0,1,float16,float16,0,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,128,0,1,float16,fp8,0,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,128,0,1,fp8,fp8,0,0.01591466615597407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,128,0,1,float16,float16,0,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,128,0,1,float16,fp8,0,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,128,0,1,fp8,fp8,0,0.01616000011563301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,128,0,1,fp8,fp8,0,0.6807253360748291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,128,0,1,float16,float16,0,1.0875519911448162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,2,128,0,1,float16,fp8,0,1.095040003458659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,128,0,1,fp8,fp8,0,1.384208043416341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,128,0,1,float16,float16,0,2.06713072458903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,128,0,1,float16,float16,0,1.0775413513183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,128,0,1,float16,fp8,0,1.0748266379038494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,2,1,128,0,1,float16,fp8,0,2.0785120328267417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,2,1,128,0,1,fp8,fp8,0,0.6678613026936849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,128,0,1,float16,float16,0,0.4906986554463704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,128,0,1,float16,fp8,0,0.4962186813354492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,2,128,0,1,fp8,fp8,0,0.3526453177134196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,128,0,1,float16,float16,0,0.525381326675415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,128,0,1,float16,fp8,0,0.48422932624816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,2,1,128,0,1,fp8,fp8,0,0.3471786578496297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,128,0,1,float16,float16,0,0.25678400198618573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,128,0,1,float16,fp8,0,0.26050132513046265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,2,128,0,1,fp8,fp8,0,0.18878400325775146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,128,0,1,float16,float16,0,0.2542346715927124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,128,0,1,float16,fp8,0,0.2546986738840739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,2,1,128,0,1,fp8,fp8,0,0.18546666701634726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,128,0,1,float16,float16,0,1.1985546747843425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,128,0,1,fp8,fp8,0,0.7793013254801432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,2,1,128,0,1,float16,fp8,0,1.213920036951701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,128,0,1,float16,float16,0,0.5715519984563192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,128,0,1,float16,fp8,0,0.5798613230387369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,2,128,0,1,fp8,fp8,0,0.4116640090942383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,128,0,1,float16,float16,0,0.5575679937998453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,128,0,1,fp8,fp8,0,0.40810132026672363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,2,1,128,0,1,float16,fp8,0,0.5648959875106812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,128,0,1,float16,float16,0,0.3211200038592021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,128,0,1,float16,fp8,0,0.31617067257563275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,2,128,0,1,fp8,fp8,0,0.23755733172098795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,128,0,1,float16,float16,0,0.3121440013249715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,128,0,1,float16,fp8,0,0.3131840030352275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,2,1,128,0,1,fp8,fp8,0,0.2307093342145284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,128,0,1,float16,float16,0,0.19951999187469482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,128,0,1,float16,fp8,0,0.2031360069910685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,2,128,0,1,fp8,fp8,0,0.14865066607793173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,128,0,1,float16,float16,0,0.1946400006612142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,128,0,1,float16,fp8,0,0.2016800045967102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,2,1,128,0,1,fp8,fp8,0,0.14194666345914206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,128,0,1,float16,float16,0,0.8730933666229248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,128,0,1,fp8,fp8,0,0.5716000000635783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,2,1,128,0,1,float16,fp8,0,0.8884267012278239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,128,0,1,float16,float16,0,0.4244053363800049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,128,0,1,fp8,fp8,0,0.2981013258298238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,128,0,1,fp8,fp8,0,0.3041920065879822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,2,128,0,1,float16,fp8,0,0.4212160110473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,128,0,1,float16,float16,0,0.4150826533635457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,2,1,128,0,1,float16,fp8,0,0.4090240001678467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,128,0,1,float16,float16,0,0.2322559952735901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,128,0,1,float16,fp8,0,0.23564799626668295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,2,128,0,1,fp8,fp8,0,0.175546665986379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,128,0,1,float16,float16,0,0.22566399971644083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,128,0,1,float16,fp8,0,0.16796799500783285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,128,0,1,float16,fp8,0,0.2260800004005432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,2,1,128,0,1,fp8,fp8,0,0.1692053278287252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,128,0,1,float16,float16,0,0.16928533713022867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,2,128,0,1,fp8,fp8,0,0.12293866276741028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,128,0,1,float16,float16,0,0.16931732495625815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,128,0,1,float16,fp8,0,0.1690559983253479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,2,1,128,0,1,fp8,fp8,0,0.12222933769226074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,128,0,1,fp8,fp8,0,0.7959413528442383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,128,0,1,float16,float16,0,1.1226133505503337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,128,0,1,float16,float16,0,0.6173280080159506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,128,0,1,fp8,fp8,0,0.40318934122721356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,2,128,0,1,float16,fp8,0,0.6086613337198893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,2,1,128,0,1,float16,fp8,0,1.117509365081787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,128,0,1,float16,float16,0,0.6004639863967896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,128,0,1,float16,fp8,0,0.6058933337529501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,2,1,128,0,1,fp8,fp8,0,0.39410666624705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,128,0,1,float16,float16,0,0.2815306584040324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,128,0,1,float16,fp8,0,0.2835093339284261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,2,128,0,1,fp8,fp8,0,0.2124320069948832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,128,0,1,float16,float16,0,0.2727573315302531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,128,0,1,float16,fp8,0,0.27374933163324994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,2,1,128,0,1,fp8,fp8,0,0.2076266606648763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,128,0,1,float16,float16,0,0.14797332882881165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,128,0,1,float16,fp8,0,0.14781333009401956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,2,128,0,1,fp8,fp8,0,0.11551466584205627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,128,0,1,float16,float16,0,0.1452906628449758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,128,0,1,float16,fp8,0,0.13718400398890176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,128,0,1,float16,fp8,0,0.1458560029665629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,2,1,128,0,1,fp8,fp8,0,0.10982933640480042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,128,0,1,float16,float16,0,0.13634133338928223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,2,128,0,1,fp8,fp8,0,0.10038933157920837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,128,0,1,float16,float16,0,0.13361066579818726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,128,0,1,float16,fp8,0,0.13486933708190918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,2,1,128,0,1,fp8,fp8,0,0.10055466492970784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,128,0,1,fp8,fp8,0,0.471562663714091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,128,0,1,float16,float16,0,0.6857386430104574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,2,1,128,0,1,float16,fp8,0,0.6892906824747721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,128,0,1,float16,float16,0,0.3342506488164266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,128,0,1,float16,fp8,0,0.33448533217112225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,2,128,0,1,fp8,fp8,0,0.2535253365834554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,128,0,1,float16,float16,0,0.3240799903869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,128,0,1,fp8,fp8,0,0.2474400003751119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,2,1,128,0,1,float16,fp8,0,0.32607465982437134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,128,0,1,float16,float16,0,0.1843679944674174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,128,0,1,float16,fp8,0,0.18580265839894614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,2,128,0,1,fp8,fp8,0,0.14310933152834573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,128,0,1,float16,fp8,0,0.1125440001487732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,128,0,1,float16,float16,0,0.1787573297818502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,128,0,1,float16,fp8,0,0.17866132656733194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,2,1,128,0,1,fp8,fp8,0,0.13666133085886636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,128,0,1,float16,fp8,0,0.10920533537864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,128,0,1,float16,float16,0,0.10989866654078166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,2,128,0,1,fp8,fp8,0,0.08640000224113464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,128,0,1,float16,float16,0,0.10965866843859355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,2,1,128,0,1,fp8,fp8,0,0.08370133241017659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,128,0,1,float16,float16,0,0.10442133744557698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,128,0,1,float16,fp8,0,0.10389332969983418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,2,128,0,1,fp8,fp8,0,0.07812800010045369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,128,0,1,float16,float16,0,0.10439999898274739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,128,0,1,float16,fp8,0,0.10460266470909119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,2,1,128,0,1,fp8,fp8,0,0.07828266421953838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,128,0,1,float16,float16,0,0.6801866690317789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,128,0,1,float16,fp8,0,0.6711466312408447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,2,1,128,0,1,fp8,fp8,0,0.5034506718317667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,128,0,1,float16,float16,0,0.3689866860707601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,128,0,1,float16,fp8,0,0.36609601974487305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,2,128,0,1,fp8,fp8,0,0.2648426691691081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,128,0,1,float16,float16,0,0.36163731416066486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,128,0,1,fp8,fp8,0,0.2540106574694316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,2,1,128,0,1,float16,fp8,0,0.3618239959081014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,128,0,1,float16,float16,0,0.1727573275566101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,128,0,1,float16,fp8,0,0.1732800006866455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,2,128,0,1,fp8,fp8,0,0.14032000303268433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,128,0,1,float16,float16,0,0.16403733690579733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,128,0,1,float16,fp8,0,0.1662346621354421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,2,1,128,0,1,fp8,fp8,0,0.1346986691157023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,128,0,1,float16,float16,0,0.0922933320204417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,128,0,1,float16,fp8,0,0.09158399701118469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,2,128,0,1,fp8,fp8,0,0.0777706652879715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,128,0,1,float16,float16,0,0.08760533730189006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,128,0,1,float16,fp8,0,0.0879306693871816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,2,1,128,0,1,fp8,fp8,0,0.07263466715812683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,128,0,1,float16,float16,0,0.07773333291212718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,128,0,1,float16,fp8,0,0.0786293347676595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,2,128,0,1,fp8,fp8,0,0.05990933378537496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,128,0,1,float16,fp8,0,0.07694399853547414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,128,0,1,float16,float16,0,0.07587733368078868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,2,1,128,0,1,fp8,fp8,0,0.059232001503308616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,128,0,1,float16,float16,0,0.07382399837176006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,128,0,1,float16,fp8,0,0.07417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,2,128,0,1,fp8,fp8,0,0.057445332407951355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,128,0,1,float16,float16,0,0.07328000168005626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,128,0,1,float16,fp8,0,0.07358400026957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,2,1,128,0,1,fp8,fp8,0,0.05588266750176748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,128,0,1,float16,float16,0,0.4220586617787679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,128,0,1,float16,fp8,0,0.418506662050883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,128,0,1,float16,float16,0,0.2182240088780721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,2,1,128,0,1,fp8,fp8,0,0.3128426671028137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,128,0,1,float16,fp8,0,0.21926933526992798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,2,128,0,1,fp8,fp8,0,0.17506666978200278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,128,0,1,float16,float16,0,0.20537600914637247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,128,0,1,float16,fp8,0,0.2055520017941793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,2,1,128,0,1,fp8,fp8,0,0.16433599591255188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,128,0,1,float16,float16,0,0.11626666784286499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,128,0,1,float16,fp8,0,0.11680000027020772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,2,128,0,1,fp8,fp8,0,0.09661333759625752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,128,0,1,float16,float16,0,0.11035199960072835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,128,0,1,float16,fp8,0,0.11229333281517029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,2,1,128,0,1,fp8,fp8,0,0.0904853343963623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,128,0,1,float16,float16,0,0.06619733572006226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,128,0,1,float16,fp8,0,0.06782400111357371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,2,128,0,1,fp8,fp8,0,0.05787200232346853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,128,0,1,float16,float16,0,0.06539733211199443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,128,0,1,float16,fp8,0,0.06562133133411407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,2,1,128,0,1,fp8,fp8,0,0.0531626691420873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,128,0,1,float16,float16,0,0.061333333452542625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,128,0,1,float16,fp8,0,0.061520000298817955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,2,128,0,1,fp8,fp8,0,0.04855999847253164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,128,0,1,float16,float16,0,0.061093335350354515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,128,0,1,float16,fp8,0,0.06066133578618368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,2,1,128,0,1,fp8,fp8,0,0.048245335618654885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,128,0,1,float16,float16,0,0.05874133110046387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,128,0,1,float16,fp8,0,0.05885333319505056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,2,128,0,1,fp8,fp8,0,0.0459146648645401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,128,0,1,float16,float16,0,0.4413226842880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,128,0,1,float16,float16,0,0.058143998185793556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,128,0,1,float16,fp8,0,0.05851200222969055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,2,1,128,0,1,fp8,fp8,0,0.04560000201066335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,128,0,1,float16,fp8,0,0.4410880009333293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,128,0,1,float16,float16,0,0.23407999674479166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,128,0,1,float16,float16,0,0.2475093404452006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,2,1,128,0,1,fp8,fp8,0,0.35368533929189044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,128,0,1,float16,fp8,0,0.24609599510828653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,2,128,0,1,fp8,fp8,0,0.19346133867899576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,128,0,1,float16,fp8,0,0.23272534211476645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,2,1,128,0,1,fp8,fp8,0,0.17968000968297324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,128,0,1,float16,float16,0,0.12084800004959106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,128,0,1,float16,fp8,0,0.1204800009727478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,2,128,0,1,fp8,fp8,0,0.10427199800809224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,128,0,1,float16,float16,0,0.11220266421635945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,128,0,1,float16,fp8,0,0.1132533351580302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,2,1,128,0,1,fp8,fp8,0,0.09689600268999736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,128,0,1,float16,float16,0,0.06550933420658112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,128,0,1,float16,fp8,0,0.06566399832566579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,2,128,0,1,fp8,fp8,0,0.06029333174228668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,128,0,1,float16,float16,0,0.0599839985370636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,128,0,1,float16,fp8,0,0.06223999957243601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,2,1,128,0,1,fp8,fp8,0,0.05401599903901418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,128,0,1,float16,float16,0,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,128,0,1,float16,fp8,0,0.04953599969546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,2,128,0,1,fp8,fp8,0,0.04186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,128,0,1,fp8,fp8,0,0.03694933404525121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,128,0,1,float16,float16,0,0.0489333321650823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,128,0,1,float16,fp8,0,0.0487306664387385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,2,1,128,0,1,fp8,fp8,0,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,128,0,1,float16,float16,0,0.046154667933781944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,2,128,0,1,float16,fp8,0,0.045850664377212524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,128,0,1,float16,float16,0,0.04560000201066335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,128,0,1,float16,fp8,0,0.0454773356517156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,2,1,128,0,1,fp8,fp8,0,0.036464000741640724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,128,0,1,float16,float16,0,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,128,0,1,float16,fp8,0,0.04368533194065094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,2,128,0,1,fp8,fp8,0,0.035061334570248924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,128,0,1,float16,float16,0,0.04359466830889384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,128,0,1,float16,float16,0,0.2850773334503174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,128,0,1,float16,fp8,0,0.28268800179163617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,128,0,1,float16,fp8,0,0.043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,2,1,128,0,1,fp8,fp8,0,0.03468266626199087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,2,1,128,0,1,fp8,fp8,0,0.2286613384882609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,128,0,1,float16,float16,0,0.1637440025806427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,128,0,1,float16,fp8,0,0.16225066781044006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,128,0,1,float16,fp8,0,0.1421440045038859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,2,128,0,1,fp8,fp8,0,0.13310399651527405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,128,0,1,float16,float16,0,0.14113600055376688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,2,1,128,0,1,fp8,fp8,0,0.1221440037091573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,128,0,1,float16,float16,0,0.08296533425649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,128,0,1,float16,fp8,0,0.0831520011027654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,2,128,0,1,fp8,fp8,0,0.07577066620190938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,128,0,1,fp8,fp8,0,0.04419200122356415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,128,0,1,float16,float16,0,0.07673599819342296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,128,0,1,float16,fp8,0,0.07751466830571492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,2,1,128,0,1,fp8,fp8,0,0.06975466509660085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,128,0,1,float16,float16,0,0.04678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,2,128,0,1,float16,fp8,0,0.04725333551565806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,128,0,1,float16,float16,0,0.04380266865094503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,128,0,1,float16,fp8,0,0.0448586642742157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,2,1,128,0,1,fp8,fp8,0,0.039434666434923805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,128,0,1,float16,float16,0,0.03915199885765711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,128,0,1,float16,fp8,0,0.04012266546487808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,2,128,0,1,fp8,fp8,0,0.035349334279696144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,128,0,1,float16,float16,0,0.038959999879201256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,128,0,1,float16,fp8,0,0.03931200007597605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,2,1,128,0,1,fp8,fp8,0,0.032655999064445496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,128,0,1,float16,float16,0,0.03690666705369949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,128,0,1,float16,fp8,0,0.037061333656311035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,128,0,1,fp8,fp8,0,0.029445332785447437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,2,128,0,1,fp8,fp8,0,0.030576000610987347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,128,0,1,float16,float16,0,0.036992001036802925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,128,0,1,float16,fp8,0,0.03718933214743932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,2,1,128,0,1,fp8,fp8,0,0.03073599934577942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,128,0,1,float16,float16,0,0.03610666592915853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,2,128,0,1,float16,fp8,0,0.035989334185918175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,128,0,1,float16,float16,0,0.03598399957021078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,128,0,1,float16,fp8,0,0.03587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,128,0,1,float16,fp8,0,0.176256000995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,2,1,128,0,1,fp8,fp8,0,0.02961066613594691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,128,0,1,float16,float16,0,0.3015200098355611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,128,0,1,float16,fp8,0,0.2955840031305949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,128,0,1,float16,float16,0,0.18028799692789713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,2,1,128,0,1,fp8,fp8,0,0.2502453327178955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,2,128,0,1,fp8,fp8,0,0.14426133036613464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,128,0,1,float16,float16,0,0.1627253293991089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,128,0,1,float16,fp8,0,0.16195199886957803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,2,1,128,0,1,fp8,fp8,0,0.1268160045146942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,128,0,1,float16,float16,0,0.09633599718411763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,128,0,1,float16,fp8,0,0.09377066294352214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,2,128,0,1,fp8,fp8,0,0.07938133180141449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,128,0,1,float16,float16,0,0.08396800359090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,128,0,1,float16,fp8,0,0.0828906645377477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,2,1,128,0,1,fp8,fp8,0,0.07082133491834004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,128,0,1,float16,float16,0,0.04957333207130432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,128,0,1,float16,fp8,0,0.049072002371152244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,2,128,0,1,fp8,fp8,0,0.04532266656557719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,128,0,1,float16,float16,0,0.04417600234349569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,128,0,1,float16,fp8,0,0.043562665581703186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,2,1,128,0,1,fp8,fp8,0,0.03826133410135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,128,0,1,float16,float16,0,0.030207999050617218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,128,0,1,float16,float16,0,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,128,0,1,float16,fp8,0,0.03403199960788091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,2,128,0,1,fp8,fp8,0,0.02972800036271413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,128,0,1,float16,float16,0,0.03281066566705704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,128,0,1,float16,fp8,0,0.03335466732581457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,2,1,128,0,1,fp8,fp8,0,0.02828266719977061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,128,0,1,float16,fp8,0,0.030565333863099415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,2,128,0,1,fp8,fp8,0,0.0258240004380544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,128,0,1,float16,float16,0,0.029680001238981884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,128,0,1,float16,fp8,0,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,2,1,128,0,1,fp8,fp8,0,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,128,0,1,float16,float16,0,0.02926933268706004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,128,0,1,float16,fp8,0,0.028912000358104706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,2,128,0,1,fp8,fp8,0,0.024645333488782246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,128,0,1,float16,float16,0,0.027914665639400482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,128,0,1,float16,fp8,0,0.027893332143624622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,2,1,128,0,1,fp8,fp8,0,0.024330665667851765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,128,0,1,float16,float16,0,0.02848000079393387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,128,0,1,float16,fp8,0,0.027978666126728058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,2,128,0,1,fp8,fp8,0,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,128,0,1,float16,float16,0,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,128,0,1,float16,fp8,0,0.0277813325325648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,2,1,128,0,1,fp8,fp8,0,0.023546665906906128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,128,0,1,float16,float16,0,0.24626133839289346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,128,0,1,float16,float16,0,0.15242133537928262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,128,0,1,float16,fp8,0,0.24158400297164917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,2,1,128,0,1,fp8,fp8,0,0.20758932828903198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,128,0,1,float16,fp8,0,0.14707733194033304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,2,128,0,1,fp8,fp8,0,0.1241386632124583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,128,0,1,float16,float16,0,0.13318933049837747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,128,0,1,float16,fp8,0,0.1309653321901957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,2,1,128,0,1,fp8,fp8,0,0.10869333148002625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,128,0,1,float16,float16,0,0.08726933598518372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,128,0,1,fp8,fp8,0,0.062165334820747375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,128,0,1,float16,fp8,0,0.08455466230710347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,2,128,0,1,fp8,fp8,0,0.0708000014225642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,128,0,1,float16,float16,0,0.0716480016708374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,2,1,128,0,1,float16,fp8,0,0.07017066578070323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,128,0,1,float16,fp8,0,0.039520000418027244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,128,0,1,float16,float16,0,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,128,0,1,float16,fp8,0,0.042725334564844765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,2,128,0,1,fp8,fp8,0,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,128,0,1,float16,float16,0,0.039605334401130676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,2,1,128,0,1,fp8,fp8,0,0.035088000198205314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,128,0,1,fp8,fp8,0,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,128,0,1,float16,float16,0,0.027957332630952198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,128,0,1,float16,fp8,0,0.02741333345572154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,2,128,0,1,fp8,fp8,0,0.02456533412138621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,128,0,1,float16,float16,0,0.026208000878492992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,128,0,1,float16,fp8,0,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,2,1,128,0,1,float16,fp8,0,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,128,0,1,float16,float16,0,0.02403733382622401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,128,0,1,float16,fp8,0,0.024186665813128155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,2,128,0,1,fp8,fp8,0,0.02146666745344798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,128,0,1,float16,float16,0,0.023034666975339253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,2,1,128,0,1,fp8,fp8,0,0.020768000433842342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,128,0,1,float16,float16,0,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,128,0,1,float16,fp8,0,0.022085333863894146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,2,128,0,1,fp8,fp8,0,0.020069333414236706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,128,0,1,float16,float16,0,0.022175999979178112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,128,0,1,float16,fp8,0,0.022250667214393616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,2,1,128,0,1,fp8,fp8,0,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,128,0,1,float16,float16,0,0.020784000555674236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,128,0,1,float16,fp8,0,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,2,128,0,1,fp8,fp8,0,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,128,0,1,float16,float16,0,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,128,0,1,float16,fp8,0,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,2,1,128,0,1,fp8,fp8,0,0.019685332973798115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,128,0,1,float16,float16,0,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,128,0,1,float16,fp8,0,0.020869334538777668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,2,128,0,1,fp8,fp8,0,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,128,0,1,float16,float16,0,0.019914666811625164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,128,0,1,float16,fp8,0,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,2,1,128,0,1,fp8,fp8,0,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,128,0,1,float16,float16,0,0.11968533198038737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,128,0,1,float16,fp8,0,0.06471466521422069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,128,0,1,float16,fp8,0,0.1172213355700175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,128,0,1,float16,float16,0,0.0828906645377477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,2,1,128,0,1,fp8,fp8,0,0.09909866253534953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,128,0,1,float16,fp8,0,0.08122133215268452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,2,128,0,1,fp8,fp8,0,0.06504000226656596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,128,0,1,float16,float16,0,0.06742933392524719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,128,0,1,float16,float16,0,0.04038399954636892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,2,1,128,0,1,fp8,fp8,0,0.05801600217819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,128,0,1,float16,fp8,0,0.03925866633653641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,2,128,0,1,fp8,fp8,0,0.03855466594298681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,128,0,1,fp8,fp8,0,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,128,0,1,float16,float16,0,0.036271999279658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,128,0,1,float16,fp8,0,0.03589333345492681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,2,1,128,0,1,fp8,fp8,0,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,128,0,1,float16,float16,0,0.025583999852339428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,2,128,0,1,float16,fp8,0,0.02518933266401291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,128,0,1,float16,float16,0,0.024085332949956257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,128,0,1,float16,fp8,0,0.02438933402299881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,2,1,128,0,1,fp8,fp8,0,0.021727999051411945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,128,0,1,float16,float16,0,0.020560000091791153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,128,0,1,float16,fp8,0,0.020869334538777668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,2,128,0,1,fp8,fp8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,128,0,1,float16,float16,0,0.020432000358899433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,128,0,1,float16,fp8,0,0.020069333414236706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,2,1,128,0,1,fp8,fp8,0,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,128,0,1,float16,float16,0,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,128,0,1,float16,fp8,0,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,2,128,0,1,fp8,fp8,0,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,128,0,1,float16,float16,0,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,128,0,1,float16,fp8,0,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,2,1,128,0,1,fp8,fp8,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,128,0,1,float16,float16,0,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,128,0,1,float16,fp8,0,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,2,128,0,1,fp8,fp8,0,0.016704000532627106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,128,0,1,float16,float16,0,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,128,0,1,float16,fp8,0,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,2,1,128,0,1,fp8,fp8,0,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,128,0,1,float16,float16,0,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,128,0,1,float16,float16,0,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,128,0,1,float16,fp8,0,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,2,128,0,1,fp8,fp8,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,128,0,1,float16,float16,0,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,128,0,1,float16,fp8,0,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,2,1,128,0,1,fp8,fp8,0,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,128,0,1,float16,fp8,0,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,2,128,0,1,fp8,fp8,0,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,128,0,1,float16,float16,0,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,128,0,1,float16,fp8,0,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,2,1,128,0,1,fp8,fp8,0,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,128,0,1,float16,float16,0,0.06574933230876923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,128,0,1,fp8,fp8,0,0.03821333249409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,128,0,1,float16,fp8,0,0.06384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,2,1,128,0,1,fp8,fp8,0,0.05794133245944977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,128,0,1,float16,float16,0,0.03999999910593033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,2,128,0,1,float16,fp8,0,0.03858133405447006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,128,0,1,float16,float16,0,0.035002666215101876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,128,0,1,float16,fp8,0,0.03472000112136205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,128,0,1,float16,float16,0,0.024197332561016083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,2,1,128,0,1,fp8,fp8,0,0.031066666046778362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,128,0,1,float16,fp8,0,0.02430933217207591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,2,128,0,1,fp8,fp8,0,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,128,0,1,float16,float16,0,0.024175999065240223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,128,0,1,float16,fp8,0,0.023818666736284893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,2,1,128,0,1,fp8,fp8,0,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,128,0,1,float16,float16,0,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,128,0,1,fp8,fp8,0,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,128,0,1,float16,fp8,0,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,2,128,0,1,fp8,fp8,0,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,128,0,1,float16,float16,0,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,2,1,128,0,1,float16,fp8,0,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,128,0,1,float16,float16,0,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,128,0,1,float16,fp8,0,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,2,128,0,1,fp8,fp8,0,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,128,0,1,float16,float16,0,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,128,0,1,float16,fp8,0,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,2,1,128,0,1,fp8,fp8,0,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,128,0,1,float16,float16,0,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,128,0,1,float16,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,2,128,0,1,fp8,fp8,0,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,128,0,1,float16,fp8,0,0.01666133354107539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,128,0,1,float16,float16,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,2,1,128,0,1,fp8,fp8,0,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,128,0,1,float16,float16,0,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,128,0,1,float16,fp8,0,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,2,128,0,1,fp8,fp8,0,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,128,0,1,float16,fp8,0,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,2,1,128,0,1,fp8,fp8,0,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,128,0,1,float16,float16,0,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,128,0,1,float16,fp8,0,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,2,128,0,1,fp8,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,128,0,1,float16,float16,0,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,128,0,1,float16,fp8,0,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,2,1,128,0,1,fp8,fp8,0,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,128,0,1,float16,float16,0,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,128,0,1,float16,fp8,0,0.016000000139077503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,2,128,0,1,fp8,fp8,0,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,128,0,1,float16,float16,0,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,128,0,1,float16,fp8,0,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,2,1,128,0,1,fp8,fp8,0,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,128,0,1,float16,float16,0,0.03701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,128,0,1,float16,fp8,0,0.035504000882307686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,2,1,128,0,1,fp8,fp8,0,0.03934400031963984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,128,0,1,float16,float16,0,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,128,0,1,float16,fp8,0,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,2,128,0,1,fp8,fp8,0,0.025936000049114227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,128,0,1,float16,float16,0,0.02474133421977361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,128,0,1,float16,fp8,0,0.024538666009902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,2,1,128,0,1,fp8,fp8,0,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,128,0,1,float16,float16,0,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,128,0,1,float16,fp8,0,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,2,128,0,1,fp8,fp8,0,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,128,0,1,float16,float16,0,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,128,0,1,float16,fp8,0,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,2,1,128,0,1,fp8,fp8,0,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,128,0,1,float16,float16,0,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,128,0,1,float16,fp8,0,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,2,128,0,1,fp8,fp8,0,0.017701332767804463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,128,0,1,float16,float16,0,0.01571200042963028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,128,0,1,float16,fp8,0,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,2,1,128,0,1,fp8,fp8,0,0.01674666628241539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,128,0,1,float16,float16,0,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,128,0,1,float16,fp8,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,2,128,0,1,fp8,fp8,0,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,128,0,1,float16,float16,0,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,128,0,1,float16,fp8,0,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,2,1,128,0,1,fp8,fp8,0,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,128,0,1,float16,float16,0,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,128,0,1,float16,fp8,0,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,128,0,1,float16,fp8,0,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,2,128,0,1,fp8,fp8,0,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,128,0,1,float16,float16,0,0.014560000350077948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,128,0,1,float16,fp8,0,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,2,1,128,0,1,fp8,fp8,0,0.016143999993801117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,128,0,1,float16,float16,0,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,2,128,0,1,fp8,fp8,0,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,128,0,1,float16,float16,0,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,128,0,1,float16,fp8,0,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,2,1,128,0,1,fp8,fp8,0,0.01594666639963786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,128,0,1,float16,float16,0,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,128,0,1,float16,fp8,0,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,2,128,0,1,fp8,fp8,0,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,128,0,1,float16,float16,0,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,128,0,1,float16,fp8,0,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,2,1,128,0,1,fp8,fp8,0,0.01618133361140887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,128,0,1,float16,float16,0,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,128,0,1,float16,fp8,0,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,2,128,0,1,fp8,fp8,0,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,128,0,1,float16,float16,0,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,128,0,1,float16,fp8,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,2,1,128,0,1,fp8,fp8,0,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,128,0,1,float16,float16,0,0.02784000088771184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,128,0,1,float16,fp8,0,0.027674667537212372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,2,1,128,0,1,fp8,fp8,0,0.032586666444937386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,128,0,1,float16,float16,0,0.019621333728233974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,128,0,1,float16,fp8,0,0.019813333948453266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,2,128,0,1,fp8,fp8,0,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,128,0,1,float16,fp8,0,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,128,0,1,float16,float16,0,0.019871999820073444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,128,0,1,float16,fp8,0,0.019621333728233974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,2,1,128,0,1,fp8,fp8,0,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,128,0,1,fp8,fp8,0,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,128,0,1,float16,float16,0,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,2,128,0,1,fp8,fp8,0,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,128,0,1,float16,float16,0,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,2,1,128,0,1,float16,fp8,0,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,128,0,1,float16,float16,0,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,128,0,1,float16,fp8,0,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,128,0,1,float16,float16,0,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,2,128,0,1,fp8,fp8,0,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,128,0,1,float16,float16,0,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,128,0,1,float16,fp8,0,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,2,1,128,0,1,fp8,fp8,0,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,128,0,1,float16,fp8,0,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,2,128,0,1,fp8,fp8,0,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,128,0,1,float16,float16,0,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,128,0,1,float16,fp8,0,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,2,1,128,0,1,fp8,fp8,0,0.016362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,128,0,1,float16,float16,0,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,128,0,1,float16,fp8,0,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,2,128,0,1,fp8,fp8,0,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,128,0,1,float16,float16,0,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,128,0,1,float16,fp8,0,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,2,1,128,0,1,fp8,fp8,0,0.01597333326935768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,128,0,1,float16,float16,0,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,128,0,1,float16,fp8,0,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,2,128,0,1,fp8,fp8,0,0.016186666985352833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,128,0,1,float16,float16,0,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,128,0,1,float16,fp8,0,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,2,1,128,0,1,fp8,fp8,0,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,128,0,1,float16,float16,0,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,128,0,1,float16,fp8,0,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,2,128,0,1,fp8,fp8,0,0.015930666277805965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,128,0,1,float16,float16,0,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,128,0,1,float16,fp8,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,2,1,128,0,1,fp8,fp8,0,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,128,0,1,float16,float16,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,128,0,1,float16,fp8,0,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,2,128,0,1,fp8,fp8,0,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,128,0,1,float16,float16,0,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,128,0,1,float16,fp8,0,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,2,1,128,0,1,fp8,fp8,0,0.01624533285697301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,128,0,1,float16,float16,0,0.024735999604066212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,128,0,1,float16,fp8,0,0.02459733436505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,2,1,128,0,1,fp8,fp8,0,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,128,0,1,float16,float16,0,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,128,0,1,float16,fp8,0,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,2,128,0,1,fp8,fp8,0,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,128,0,1,float16,float16,0,0.017770666629076004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,128,0,1,float16,fp8,0,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,2,1,128,0,1,fp8,fp8,0,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,128,0,1,float16,float16,0,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,128,0,1,float16,fp8,0,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,2,128,0,1,fp8,fp8,0,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,128,0,1,float16,float16,0,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,128,0,1,float16,fp8,0,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,2,1,128,0,1,fp8,fp8,0,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,128,0,1,float16,float16,0,0.01458666721979777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,128,0,1,float16,fp8,0,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,2,128,0,1,fp8,fp8,0,0.01618133361140887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,128,0,1,float16,float16,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,128,0,1,float16,fp8,0,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,2,1,128,0,1,fp8,fp8,0,0.016186666985352833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,128,0,1,float16,float16,0,0.014138666292031607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,128,0,1,float16,fp8,0,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,2,128,0,1,fp8,fp8,0,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,128,0,1,fp8,fp8,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,128,0,1,float16,float16,0,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,128,0,1,float16,fp8,0,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,2,1,128,0,1,fp8,fp8,0,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,128,0,1,float16,float16,0,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,2,128,0,1,float16,fp8,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,128,0,1,float16,float16,0,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,128,0,1,float16,fp8,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,2,1,128,0,1,fp8,fp8,0,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,128,0,1,float16,fp8,0,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,2,128,0,1,fp8,fp8,0,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,128,0,1,float16,float16,0,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,128,0,1,float16,fp8,0,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,2,1,128,0,1,fp8,fp8,0,0.016074666132529575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,128,0,1,float16,float16,0,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,128,0,1,float16,fp8,0,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,2,128,0,1,fp8,fp8,0,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,128,0,1,float16,float16,0,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,128,0,1,float16,fp8,0,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,2,1,128,0,1,fp8,fp8,0,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,128,0,1,float16,float16,0,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,128,0,1,float16,fp8,0,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,2,128,0,1,fp8,fp8,0,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,128,0,1,float16,float16,0,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,128,0,1,float16,fp8,0,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,2,1,128,0,1,fp8,fp8,0,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,128,0,1,float16,float16,0,0.48635733127593994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,128,0,1,float16,fp8,0,0.48792533079783124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,1,1,128,0,1,fp8,fp8,0,0.3489760160446167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,128,0,1,float16,float16,0,0.2541973392168681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,128,0,1,float16,fp8,0,0.25244800249735516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,1,1,128,0,1,fp8,fp8,0,0.18548800547917685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,128,0,1,float16,float16,0,0.24396799008051553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,128,0,1,float16,fp8,0,0.24525866905848184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,1,1,128,0,1,fp8,fp8,0,0.17267199357350668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,128,0,1,float16,float16,0,0.3143306573232015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,128,0,1,float16,fp8,0,0.31778132915496826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,1,1,128,0,1,fp8,fp8,0,0.23017066717147827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,128,0,1,float16,float16,0,0.19428267081578574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,128,0,1,float16,fp8,0,0.19430400927861533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,1,1,128,0,1,fp8,fp8,0,0.1437226633230845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,128,0,1,float16,float16,0,0.18488534291585287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,128,0,1,float16,fp8,0,0.18532266219456991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,128,0,1,float16,float16,0,0.16452266772588095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,1,1,128,0,1,fp8,fp8,0,0.1330933372179667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,128,0,1,float16,float16,0,0.2295786738395691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,128,0,1,float16,fp8,0,0.2290560007095337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,1,1,128,0,1,fp8,fp8,0,0.1714186668395996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,128,0,1,float16,fp8,0,0.16431466738382974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,1,1,128,0,1,fp8,fp8,0,0.1186133325099945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,128,0,1,float16,float16,0,0.15567466616630554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,128,0,1,float16,float16,0,0.27769599358240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,128,0,1,float16,fp8,0,0.27799999713897705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,128,0,1,float16,fp8,0,0.15705600380897522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,1,1,128,0,1,fp8,fp8,0,0.1127786636352539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,1,1,128,0,1,fp8,fp8,0,0.21040532986323038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,128,0,1,float16,float16,0,0.14409599701563516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,128,0,1,float16,fp8,0,0.14638400077819824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,1,1,128,0,1,fp8,fp8,0,0.11346133550008138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,128,0,1,float16,float16,0,0.13089066743850708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,128,0,1,float16,fp8,0,0.1340053379535675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,1,1,128,0,1,fp8,fp8,0,0.09668266773223877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,128,0,1,float16,float16,0,0.1279093325138092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,128,0,1,float16,fp8,0,0.12781332929929098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,1,1,128,0,1,fp8,fp8,0,0.09250133236249287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,128,0,1,float16,float16,0,0.18316799402236938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,128,0,1,float16,fp8,0,0.18368534247080484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,1,1,128,0,1,fp8,fp8,0,0.14125333229700723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,128,0,1,float16,float16,0,0.1095199982325236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,128,0,1,float16,fp8,0,0.10863999525705974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,1,1,128,0,1,fp8,fp8,0,0.08558932940165202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,128,0,1,float16,float16,0,0.10191466410954793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,128,0,1,float16,fp8,0,0.10180800159772237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,1,1,128,0,1,fp8,fp8,0,0.0754559983809789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,128,0,1,float16,float16,0,0.09880533814430237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,128,0,1,float16,fp8,0,0.09963732957839966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,1,1,128,0,1,fp8,fp8,0,0.07314133147398631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,128,0,1,float16,fp8,0,0.09290666381518047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,128,0,1,float16,float16,0,0.17300266027450562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,128,0,1,float16,fp8,0,0.1728960076967875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,1,1,128,0,1,fp8,fp8,0,0.1400320033232371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,128,0,1,float16,float16,0,0.09078933795293172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,1,1,128,0,1,fp8,fp8,0,0.07782933115959167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,128,0,1,float16,float16,0,0.07627200086911519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,128,0,1,float16,fp8,0,0.07739200194676717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,1,1,128,0,1,fp8,fp8,0,0.06044266621271769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,128,0,1,float16,float16,0,0.07271466652552287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,128,0,1,float16,fp8,0,0.07257600128650665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,1,1,128,0,1,fp8,fp8,0,0.05519466598828634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,128,0,1,float16,float16,0,0.07060266534487407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,128,0,1,float16,fp8,0,0.07076266904671986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,1,1,128,0,1,fp8,fp8,0,0.05346133311589559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,128,0,1,float16,float16,0,0.11715733011563619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,128,0,1,float16,fp8,0,0.11592533191045125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,1,1,128,0,1,fp8,fp8,0,0.09570667147636414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,128,0,1,float16,float16,0,0.06644266843795776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,128,0,1,float16,fp8,0,0.0671253353357315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,1,1,128,0,1,fp8,fp8,0,0.05738666653633118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,128,0,1,float16,float16,0,0.0591839998960495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,128,0,1,float16,fp8,0,0.06036800146102905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,1,1,128,0,1,fp8,fp8,0,0.04762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,128,0,1,float16,float16,0,0.057189335425694786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,128,0,1,float16,fp8,0,0.05748266478379568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,1,1,128,0,1,fp8,fp8,0,0.044591998060544334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,128,0,1,float16,float16,0,0.05574933191140493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,128,0,1,float16,fp8,0,0.05600533386071523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,1,1,128,0,1,fp8,fp8,0,0.043151999513308205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,128,0,1,float16,float16,0,0.11931733290354411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,128,0,1,float16,float16,0,0.04840533435344696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,128,0,1,float16,fp8,0,0.12043199936548869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,1,1,128,0,1,fp8,fp8,0,0.1027893324693044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,128,0,1,float16,float16,0,0.06364266574382782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,128,0,1,float16,fp8,0,0.06513066589832306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,1,1,128,0,1,fp8,fp8,0,0.059450666109720864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,128,0,1,float16,fp8,0,0.04929600159327189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,1,1,128,0,1,fp8,fp8,0,0.04185600082079569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,128,0,1,float16,float16,0,0.04387733340263367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,128,0,1,float16,fp8,0,0.04506133496761322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,1,1,128,0,1,fp8,fp8,0,0.036570665736993156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,128,0,1,float16,float16,0,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,128,0,1,float16,fp8,0,0.04309333364168803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,1,1,128,0,1,fp8,fp8,0,0.034527999659379326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,128,0,1,float16,float16,0,0.04201066493988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,128,0,1,float16,fp8,0,0.04192000130812327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,1,1,128,0,1,fp8,fp8,0,0.03346666693687439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,128,0,1,float16,float16,0,0.08455466230710347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,128,0,1,float16,fp8,0,0.08523733417193095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,1,1,128,0,1,fp8,fp8,0,0.07798400024573009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,128,0,1,float16,float16,0,0.04752533137798309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,128,0,1,float16,fp8,0,0.04717333118120829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,1,1,128,0,1,fp8,fp8,0,0.04452266792456309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,128,0,1,float16,float16,0,0.039066667358080544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,128,0,1,float16,fp8,0,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,1,1,128,0,1,fp8,fp8,0,0.034330666065216064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,128,0,1,float16,float16,0,0.03701333453257879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,128,0,1,float16,fp8,0,0.037263999382654824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,1,1,128,0,1,fp8,fp8,0,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,128,0,1,float16,float16,0,0.03532266616821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,128,0,1,float16,fp8,0,0.03621866554021835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,1,1,128,0,1,fp8,fp8,0,0.029525332152843475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,128,0,1,float16,float16,0,0.034858666360378265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,128,0,1,float16,fp8,0,0.09436800082524617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,128,0,1,float16,fp8,0,0.03425599883000056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,1,1,128,0,1,fp8,fp8,0,0.028421332438786823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,128,0,1,float16,float16,0,0.0974079966545105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,1,1,128,0,1,fp8,fp8,0,0.0798880010843277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,128,0,1,float16,float16,0,0.049925332268079124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,128,0,1,float16,fp8,0,0.048991998036702476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,1,1,128,0,1,fp8,fp8,0,0.04582933088143667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,128,0,1,float16,float16,0,0.033301333586374916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,128,0,1,float16,fp8,0,0.033813332517941795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,128,0,1,float16,float16,0,0.028586665789286297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,1,1,128,0,1,fp8,fp8,0,0.029066666960716248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,128,0,1,float16,float16,0,0.030378667016824085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,128,0,1,float16,fp8,0,0.030437332888444264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,1,1,128,0,1,fp8,fp8,0,0.025888000925381977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,128,0,1,float16,fp8,0,0.028394666810830433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,1,1,128,0,1,fp8,fp8,0,0.02430933217207591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,128,0,1,float16,float16,0,0.028538666665554047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,128,0,1,float16,fp8,0,0.029050665597120922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,1,1,128,0,1,fp8,fp8,0,0.024122667809327442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,128,0,1,float16,float16,0,0.026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,128,0,1,float16,fp8,0,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,1,1,128,0,1,fp8,fp8,0,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,128,0,1,float16,float16,0,0.08750399947166443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,128,0,1,float16,fp8,0,0.08425066868464152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,128,0,1,float16,float16,0,0.028037334481875103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,1,1,128,0,1,fp8,fp8,0,0.07102933526039124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,128,0,1,float16,fp8,0,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,128,0,1,float16,float16,0,0.04390933116277059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,128,0,1,float16,fp8,0,0.04159999887148539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,1,1,128,0,1,fp8,fp8,0,0.04141333450873693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,128,0,1,float16,fp8,0,0.022634667654832203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,128,0,1,float16,fp8,0,0.028581333657105763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,1,1,128,0,1,fp8,fp8,0,0.02478400121132533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,128,0,1,float16,float16,0,0.023749334116776783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,1,1,128,0,1,fp8,fp8,0,0.02181866765022278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,128,0,1,float16,float16,0,0.020319999506076176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,128,0,1,float16,float16,0,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,1,1,128,0,1,fp8,fp8,0,0.019573333362738293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,128,0,1,float16,float16,0,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,128,0,1,float16,fp8,0,0.022229333718617756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,1,1,128,0,1,fp8,fp8,0,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,128,0,1,float16,fp8,0,0.02046400060256322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,1,1,128,0,1,fp8,fp8,0,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,128,0,1,float16,float16,0,0.02014933278163274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,128,0,1,float16,fp8,0,0.02035733312368393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,1,1,128,0,1,fp8,fp8,0,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,128,0,1,float16,float16,0,0.04025600105524063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,128,0,1,float16,fp8,0,0.038949333131313324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,1,1,128,0,1,fp8,fp8,0,0.03872533390919367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,128,0,1,float16,float16,0,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,128,0,1,float16,fp8,0,0.02565866708755493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,1,1,128,0,1,fp8,fp8,0,0.02266666789849599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,128,0,1,float16,float16,0,0.020309332758188248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,128,0,1,float16,fp8,0,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,1,1,128,0,1,fp8,fp8,0,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,128,0,1,float16,float16,0,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,128,0,1,float16,fp8,0,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,1,1,128,0,1,fp8,fp8,0,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,128,0,1,float16,float16,0,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,128,0,1,float16,float16,0,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,128,0,1,float16,fp8,0,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,1,1,128,0,1,fp8,fp8,0,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,128,0,1,float16,float16,0,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,128,0,1,float16,fp8,0,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,1,1,128,0,1,fp8,fp8,0,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,128,0,1,float16,fp8,0,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,1,1,128,0,1,fp8,fp8,0,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,128,0,1,float16,float16,0,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,128,0,1,float16,fp8,0,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,1,1,128,0,1,fp8,fp8,0,0.016282666474580765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,128,0,1,float16,float16,0,0.024703999360402424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,128,0,1,float16,fp8,0,0.024336000283559162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,128,0,1,float16,fp8,0,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,1,1,128,0,1,fp8,fp8,0,0.0229120006163915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,128,0,1,float16,float16,0,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,128,0,1,float16,fp8,0,0.019637333850065868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,128,0,1,float16,float16,0,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,1,1,128,0,1,fp8,fp8,0,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,128,0,1,float16,float16,0,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,1,1,128,0,1,fp8,fp8,0,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,128,0,1,float16,float16,0,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,128,0,1,fp8,fp8,0,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,128,0,1,float16,fp8,0,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,1,1,128,0,1,fp8,fp8,0,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,128,0,1,float16,fp8,0,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,1,1,128,0,1,fp8,fp8,0,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,128,0,1,float16,float16,0,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,1,1,128,0,1,float16,fp8,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,128,0,1,fp8,fp8,0,0.016352000335852306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,128,0,1,float16,float16,0,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,128,0,1,float16,fp8,0,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,1,1,128,0,1,fp8,fp8,0,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,128,0,1,float16,float16,0,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,1,1,128,0,1,float16,fp8,0,0.015978666643301647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,128,0,1,float16,float16,0,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,128,0,1,float16,fp8,0,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,1,1,128,0,1,fp8,fp8,0,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,128,0,1,float16,float16,0,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,128,0,1,float16,fp8,0,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,1,1,128,0,1,fp8,fp8,0,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,128,0,1,float16,float16,0,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,128,0,1,float16,fp8,0,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,1,1,128,0,1,fp8,fp8,0,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,128,0,1,float16,float16,0,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,128,0,1,float16,fp8,0,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,1,1,128,0,1,fp8,fp8,0,0.01607999950647354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,128,0,1,float16,float16,0,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,128,0,1,float16,fp8,0,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,128,0,1,float16,float16,0,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,1,1,128,0,1,fp8,fp8,0,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,128,0,1,float16,float16,0,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,128,0,1,float16,float16,0,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,128,0,1,float16,fp8,0,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,1,1,128,0,1,fp8,fp8,0,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,128,0,1,float16,fp8,0,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,1,1,128,0,1,fp8,fp8,0,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,128,0,1,float16,fp8,0,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,1,1,128,0,1,fp8,fp8,0,0.015919999529918034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,128,0,1,float16,fp8,0,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,128,0,1,float16,float16,0,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,128,0,1,float16,fp8,0,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,1,1,128,0,1,fp8,fp8,0,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,128,0,1,float16,float16,0,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,1,1,128,0,1,fp8,fp8,0,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,128,0,1,float16,fp8,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,1,1,128,0,1,fp8,fp8,0,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,128,0,1,float16,float16,0,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,128,0,1,float16,fp8,0,0.014554666976133982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,1,1,128,0,1,fp8,fp8,0,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,128,0,1,float16,float16,0,0.01393066719174385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,128,0,1,float16,fp8,0,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,1,1,128,0,1,fp8,fp8,0,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,128,0,1,float16,float16,0,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,128,0,1,float16,fp8,0,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,1,1,128,0,1,fp8,fp8,0,0.01599466676513354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,128,0,1,float16,float16,0,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,128,0,1,float16,fp8,0,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,1,1,128,0,1,fp8,fp8,0,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,128,0,1,float16,float16,0,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,128,0,1,fp8,fp8,0,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,128,0,1,float16,fp8,0,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,1,1,128,0,1,fp8,fp8,0,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,128,0,1,float16,float16,0,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,1,1,128,0,1,float16,fp8,0,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,128,0,1,fp8,fp8,0,0.016336000214020412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,128,0,1,float16,float16,0,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,128,0,1,float16,fp8,0,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,1,1,128,0,1,fp8,fp8,0,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,128,0,1,float16,float16,0,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,1,1,128,0,1,float16,fp8,0,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,128,0,1,float16,float16,0,0.013978666315476099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,128,0,1,float16,fp8,0,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,1,1,128,0,1,fp8,fp8,0,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,128,0,1,float16,float16,0,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,128,0,1,float16,fp8,0,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,1,1,128,0,1,fp8,fp8,0,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,128,0,1,float16,float16,0,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,128,0,1,float16,fp8,0,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,1,1,128,0,1,fp8,fp8,0,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,128,0,1,float16,float16,0,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,128,0,1,float16,fp8,0,0.014261333892742792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,1,1,128,0,1,fp8,fp8,0,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,128,0,1,float16,float16,0,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,128,0,1,float16,fp8,0,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,1,1,128,0,1,fp8,fp8,0,0.01580799991885821
